From 7d6fbf9c7bc6cb63605acb778d5b09682af4ccf8 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 25 Jul 2026 10:32:26 +0200 Subject: [PATCH 001/391] conn: implement thread-local connection struct freelist recycler nxt_conn_free() pushes the freed nxt_conn_t onto its engine's free_connections list and nxt_conn_create() pops and re-zeroes a recycled struct when one is available, avoiding a pool allocation per accept on the hot path. Full-struct nxt_memzero on reuse mirrors the nxt_mp_zget zeroing, so recycled conns carry no stale timers, queue links, or state. The freelist recycles only structs backed by the persistent engine->mem_pool, which outlives every per-connection pool. When engine->mem_pool is NULL -- a real state, since nxt_runtime.c creates it with nxt_mp_create() and no NULL check, so an OOM at engine setup leaves it NULL -- nxt_conn_create() allocates the struct from the per-connection pool mp as a non-recyclable struct and nxt_conn_free() never pushes it: the struct dies with its pool, leaving no dangling freelist entry. In the recyclable case the struct does not live in mp, so releasing mp never frees the pushed struct -- the push-before-release window is closed by construction. nxt_conn_free() also asserts the conn is freed on the engine that created it (c->task.thread == task->thread): a cross-engine free would dangle when the owning engine's mem_pool is destroyed at listen_threads churn. The freelist pins a per-engine high-water mark of recycled structs (~432 bytes each) until worker-thread exit, bounded by peak concurrency per engine and freed wholesale with engine->mem_pool. test_conn_recycle.py drives short-lived connections across multiple listen_threads engines so each engine's freelist is populated, then lowers listen_threads to destroy those engines (freeing their populated freelists via engine->mem_pool teardown) and raises it again, repeatedly, verifying the router keeps serving correct bodies throughout. Added to the ASan sanitize subset, where a corrupted recycle or a teardown UAF surfaces as a worker crash rather than passing silently. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SwiBzuZGie84r2WFZpE3x7 --- .github/workflows/sanitize.yml | 9 ++++- src/nxt_conn.c | 65 ++++++++++++++++++++++++++---- src/nxt_conn.h | 1 + src/nxt_event_engine.h | 2 + test/test_conn_recycle.py | 73 ++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 test/test_conn_recycle.py diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 58887a126..7d7bee412 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -64,8 +64,12 @@ jobs: # unitd master and self-skips without it. It also makes conftest # rmtree the temp dir on teardown, which is fine for these tests. pytest_opts: "--restart" - # test_static is module-free; the graceful/listener drivers need - # the python module, so the shared teardown coverage lives here. + # test_static and test_conn_recycle are module-free; the + # graceful/listener drivers need the python module, so the shared + # teardown coverage lives here. test_conn_recycle exercises the + # connection-struct freelist recycler under ASan (recycle churn + + # engine teardown at listen_threads reconfig). + # # test_conn_task_race drives a send-timeout mid-stream RST abort # straddling keep-alive reuse (freeunit#156): pre-fix a deferred # conn write item dereferenced a freed request-pool task, which @@ -91,6 +95,7 @@ jobs: test/test_conn_task_race.py test/test_proxy.py test/test_chunked.py + test/test_conn_recycle.py - runtime: php apt: "" configure: php diff --git a/src/nxt_conn.c b/src/nxt_conn.c index b4950d17a..79d3cf813 100644 --- a/src/nxt_conn.c +++ b/src/nxt_conn.c @@ -45,9 +45,35 @@ nxt_conn_create(nxt_mp_t *mp, nxt_task_t *task) nxt_conn_t *c; nxt_thread_t *thr; - c = nxt_mp_zget(mp, sizeof(nxt_conn_t)); - if (nxt_slow_path(c == NULL)) { - return NULL; + thr = nxt_thread(); + + /* + * The freelist only recycles connection structs backed by the persistent + * engine->mem_pool, so a recycled struct outlives every per-connection + * pool. When engine->mem_pool is NULL (e.g. an OOM at engine setup) the + * struct is allocated from the per-connection pool mp and is therefore + * NOT recyclable; nxt_conn_free() must not push it onto the freelist. + */ + + if (thr->engine->mem_pool != NULL) { + + if (thr->engine->free_connections != NULL) { + c = thr->engine->free_connections; + thr->engine->free_connections = c->next; + nxt_memzero(c, sizeof(nxt_conn_t)); + + } else { + c = nxt_mp_zget(thr->engine->mem_pool, sizeof(nxt_conn_t)); + if (nxt_slow_path(c == NULL)) { + return NULL; + } + } + + } else { + c = nxt_mp_zget(mp, sizeof(nxt_conn_t)); + if (nxt_slow_path(c == NULL)) { + return NULL; + } } c->mem_pool = mp; @@ -63,7 +89,6 @@ nxt_conn_create(nxt_mp_t *mp, nxt_task_t *task) c->log.ident = nxt_task_next_ident(); } - thr = nxt_thread(); thr->engine->connections++; c->task.thread = thr; @@ -104,12 +129,38 @@ nxt_conn_create(nxt_mp_t *mp, nxt_task_t *task) void nxt_conn_free(nxt_task_t *task, nxt_conn_t *c) { - nxt_mp_t *mp; + nxt_mp_t *mp; + nxt_event_engine_t *engine; - task->thread->engine->connections--; + engine = task->thread->engine; + engine->connections--; mp = c->mem_pool; - nxt_mp_release(mp); + + /* + * Only recycle structs backed by the persistent engine->mem_pool. A + * freelist entry must outlive every per-connection pool, so when + * engine->mem_pool is NULL the struct lives in mp (== c->mem_pool) and + * must die with it -- pushing it would leave a dangling pointer on the + * freelist once mp is released below. In the recycled case the struct + * does not live in mp, so releasing mp does not free the pushed struct. + * + * A conn must be freed on the engine that created it: recycling a struct + * backed by another engine's mem_pool would dangle once that engine is + * torn down. The pre-existing connections-- accounting already relies on + * this; assert it so the freelist can too. + */ + + nxt_assert(c->task.thread == task->thread); + + if (engine->mem_pool != NULL) { + c->next = engine->free_connections; + engine->free_connections = c; + } + + if (mp != NULL) { + nxt_mp_release(mp); + } } diff --git a/src/nxt_conn.h b/src/nxt_conn.h index c193713ea..89cff7da4 100644 --- a/src/nxt_conn.h +++ b/src/nxt_conn.h @@ -193,6 +193,7 @@ struct nxt_conn_s { uint8_t tcp_nodelay; /* 1 bit */ nxt_queue_link_t link; + nxt_conn_t *next; }; diff --git a/src/nxt_event_engine.h b/src/nxt_event_engine.h index 8f5666851..09b46c6f7 100644 --- a/src/nxt_event_engine.h +++ b/src/nxt_event_engine.h @@ -484,6 +484,8 @@ struct nxt_event_engine_s { nxt_queue_t active_connections; nxt_array_t *mem_cache; + nxt_conn_t *free_connections; + nxt_atomic_uint_t accepted_conns_cnt; nxt_atomic_uint_t idle_conns_cnt; nxt_atomic_uint_t active_conns_cnt; diff --git a/test/test_conn_recycle.py b/test/test_conn_recycle.py new file mode 100644 index 000000000..1ee826569 --- /dev/null +++ b/test/test_conn_recycle.py @@ -0,0 +1,73 @@ +import socket +from pathlib import Path + +import pytest + +from unit.applications.proto import ApplicationProto + +client = ApplicationProto() + +BODY = '0123456789' + + +@pytest.fixture(autouse=True) +def setup_method_fixture(temp_dir, skip_fds_check): + # Thread-count changes intentionally add and remove each engine's epoll, + # signalfd, and eventfd descriptors, so the generic router FD baseline is + # not meaningful for this test. The recycler itself owns no descriptors. + skip_fds_check(router=True) + + Path(f'{temp_dir}/assets').mkdir(parents=True) + Path(f'{temp_dir}/assets/index.html').write_text(BODY, encoding='utf-8') + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"share": f'{temp_dir}/assets$uri'}}], + "settings": {"listen_threads": 4}, + } + ) + + yield + + # The no-restart suite preserves /settings between tests. + assert 'success' in client.conf_delete('settings/listen_threads') + + +def _churn(n): + # Each short-lived connection is a nxt_conn_create (freelist pop) + + # nxt_conn_free (freelist push) cycle; spread across the listen_threads + # engines this fills each engine's connection freelist. A corrupted + # recycle returns a wrong/truncated body or, under the sanitizer, crashes + # the worker. + for i in range(n): + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(5) + try: + sock.sendall( + b'GET / HTTP/1.1\r\nHost: localhost\r\n' + b'Connection: close\r\n\r\n' + ) + data = b'' + while BODY.encode() not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + assert BODY.encode() in data, f'body on connection {i}' + finally: + sock.close() + + +def test_conn_recycle_across_thread_churn(): + # Populate the per-engine connection freelists. + _churn(120) + + # Lowering listen_threads destroys worker engines, freeing their populated + # connection freelists (engine->mem_pool teardown); raising it recreates + # them. Repeat so structs are recycled, then torn down with the engine. + for _ in range(2): + assert 'success' in client.conf('1', 'settings/listen_threads') + _churn(60) + assert 'success' in client.conf('4', 'settings/listen_threads') + _churn(60) From 82b0b30c3cc68d808c14f38ad284e84000d77c91 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 26 Jul 2026 03:40:30 +0200 Subject: [PATCH 002/391] conn: publish recycled connection structs only when the engine is quiescent The freelist handed a freed nxt_conn_t straight back to nxt_conn_create(), which re-zeroes it. A connection can be freed while work items that reference it are still queued: nxt_h1p_peer_close() sets block_read and block_write precisely so such items bail out, then reaches nxt_conn_free() synchronously on its fd == -1 path, with no queue drain in between. Reusing the struct there clears those guards from under the queued item, which then proceeds against an unrelated live connection. The connection-task assertions added by #157/#160 do not catch this: nxt_conn_create() rebinds c->socket.task to &c->task, the same embedded address, so the stale item satisfies them. Nor does the current queue priority make it safe by design -- it happens that nxt_event_engine_queue_pop() polls fast_work_queue before the close queues, so on the deferred-close path the stale item usually runs first. That is a scheduler implementation detail, not a property the recycler guarantees. Make it structural instead. nxt_conn_free() parks the struct on a new engine->pending_connections list, and nxt_conn_recycle_pending() moves parked structs to engine->free_connections from the one point in nxt_event_engine_start() where nxt_event_engine_queue_pop() reports every work queue empty. A struct therefore becomes reusable only once nothing is left queued that could reference it, on every free path including the synchronous one. The block_read / block_write guards keep working exactly as before -- a stale item still runs before promotion and still bails; what changes is that the guards can no longer be erased while still needed. Timers are the second hazard and are handled here too. Callers only nxt_timer_disable() a connection's timers, which clears the enabled bit but leaves the node in the engine's rbtree, so re-zeroing the struct would corrupt that tree. nxt_conn_free() now nxt_timer_delete()s both, and since that removal can be deferred as a queued change, promotion additionally requires each timer to be out of the tree with no queued change. A struct whose timers have not settled stays parked and is retried next cycle. The optimisation is unaffected. Instrumented against test_conn_recycle.py, recycled-struct reuse is identical with and without this change (374 reuses either way; 398 of 403 freed structs promoted, the remainder simply still parked at teardown). Verified on a --debug build so the #157/#160 assertions are live: test_conn_recycle, test_proxy, test_static, test_php_application and test_idle_close_wait give 72 passed, 21 skipped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QgUyzdRHR4e4q6THZZd9Sa --- src/nxt_conn.c | 69 ++++++++++++++++++++++++++++++++++++++++-- src/nxt_conn.h | 1 + src/nxt_event_engine.c | 6 ++++ src/nxt_event_engine.h | 1 + 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/nxt_conn.c b/src/nxt_conn.c index 79d3cf813..145395cbe 100644 --- a/src/nxt_conn.c +++ b/src/nxt_conn.c @@ -154,8 +154,31 @@ nxt_conn_free(nxt_task_t *task, nxt_conn_t *c) nxt_assert(c->task.thread == task->thread); if (engine->mem_pool != NULL) { - c->next = engine->free_connections; - engine->free_connections = c; + /* + * Take both timers out of the engine's timer machinery. Callers only + * ever nxt_timer_disable() them, which clears the enabled bit but + * leaves the node in the engine's rbtree -- re-zeroing the struct on + * reuse would then corrupt that tree. nxt_timer_delete() removes it, + * but the removal can be deferred as a queued change, so quiescence is + * rechecked in nxt_conn_recycle_pending() before the struct is handed + * out again. + */ + nxt_timer_delete(engine, &c->read_timer); + nxt_timer_delete(engine, &c->write_timer); + + /* + * Park the struct rather than publishing it to the freelist directly. + * A conn can be freed while work items that reference it are still + * queued -- nxt_h1p_peer_close() sets block_read/block_write for + * exactly that reason, then reaches nxt_conn_free() synchronously on + * its fd == -1 path. Handing the struct out now would let + * nxt_conn_create()'s re-zero clear those guards from under the queued + * item, which would then act on an unrelated live connection. + * nxt_conn_recycle_pending() promotes the struct only once the engine + * has drained every work queue, so no queued item can still reach it. + */ + c->next = engine->pending_connections; + engine->pending_connections = c; } if (mp != NULL) { @@ -164,6 +187,48 @@ nxt_conn_free(nxt_task_t *task, nxt_conn_t *c) } +nxt_inline nxt_bool_t +nxt_conn_timer_quiescent(nxt_timer_t *timer) +{ + return !nxt_timer_is_in_tree(timer) + && !timer->queued + && timer->change == NXT_TIMER_NO_CHANGE; +} + + +/* + * Publish parked connection structs to the reusable freelist. Called from the + * event engine once nxt_event_engine_queue_pop() reports every work queue + * empty: at that point no queued work item can reference any parked struct, so + * reuse cannot race a handler that is still to run. A struct whose timers have + * not settled yet stays parked and is retried on a later cycle. + */ + +void +nxt_conn_recycle_pending(nxt_event_engine_t *engine) +{ + nxt_conn_t *c, *next, **link; + + link = &engine->pending_connections; + + for (c = *link; c != NULL; c = next) { + next = c->next; + + if (nxt_conn_timer_quiescent(&c->read_timer) + && nxt_conn_timer_quiescent(&c->write_timer)) + { + *link = next; + + c->next = engine->free_connections; + engine->free_connections = c; + + } else { + link = &c->next; + } + } +} + + void nxt_conn_timer(nxt_event_engine_t *engine, nxt_conn_t *c, const nxt_conn_state_t *state, nxt_timer_t *timer) diff --git a/src/nxt_conn.h b/src/nxt_conn.h index 89cff7da4..f2ef4932f 100644 --- a/src/nxt_conn.h +++ b/src/nxt_conn.h @@ -245,6 +245,7 @@ struct nxt_conn_s { NXT_EXPORT nxt_conn_t *nxt_conn_create(nxt_mp_t *mp, nxt_task_t *task); NXT_EXPORT void nxt_conn_free(nxt_task_t *task, nxt_conn_t *c); +NXT_EXPORT void nxt_conn_recycle_pending(nxt_event_engine_t *engine); NXT_EXPORT void nxt_conn_close(nxt_event_engine_t *engine, nxt_conn_t *c); NXT_EXPORT void nxt_conn_timer(nxt_event_engine_t *engine, nxt_conn_t *c, diff --git a/src/nxt_event_engine.c b/src/nxt_event_engine.c index 9a37b34c4..6269974fa 100644 --- a/src/nxt_event_engine.c +++ b/src/nxt_event_engine.c @@ -547,6 +547,12 @@ nxt_event_engine_start(nxt_event_engine_t *engine) handler = nxt_event_engine_queue_pop(engine, &task, &obj, &data); if (handler == NULL) { + /* + * Every work queue is empty here, so no queued item can still + * reference a connection struct freed during this drain. This + * is the only point at which parked structs become reusable. + */ + nxt_conn_recycle_pending(engine); break; } diff --git a/src/nxt_event_engine.h b/src/nxt_event_engine.h index 09b46c6f7..cff72022f 100644 --- a/src/nxt_event_engine.h +++ b/src/nxt_event_engine.h @@ -485,6 +485,7 @@ struct nxt_event_engine_s { nxt_array_t *mem_cache; nxt_conn_t *free_connections; + nxt_conn_t *pending_connections; nxt_atomic_uint_t accepted_conns_cnt; nxt_atomic_uint_t idle_conns_cnt; From 9eaffef544d6d07cdf817ddc1417139ba2383e9c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 26 Jul 2026 04:11:00 +0200 Subject: [PATCH 003/391] conn: untrack on free, and scope the recycle barrier claim accurately Addresses the careful review of b081ee39. Untrack in nxt_conn_free(). The commit deleted the timers because callers only nxt_timer_disable() them, leaving a live node inside a struct that nxt_conn_create() re-zeroes. c->link is the identical hazard and was left open: while c->idle != NXT_CONN_TRACK_NONE the conn is a live node on engine->idle_connections or active_connections, and nxt_conn_free() relies entirely on nxt_conn_close_handler() having untracked it. All five free sites are clean today -- confirmed here, the new assertion did not fire across 240 connections with listen_threads churn -- but the recycler changes the failure mode of a future miss from an ASan-visible link into released pool memory to a silently zeroed node with drifting idle/active counters. nxt_conn_untrack() is idempotent, so assert the invariant and untrack anyway: caught in debug, harmless in release. nxt_controller_conn_free() already keeps exactly this backstop. Both the untrack and the timer deletes now run outside the recycling branch. On the non-recycling path the struct dies with mp, so a node left behind dangles into released memory rather than being re-zeroed -- the same defect with a worse outcome, and the two calls cost nothing there. Scope the barrier claim. The comments said "every work queue is empty", which is what makes the design safe, but nxt_event_engine_queue_pop() only scans fast_work_queue..close_work_queue; engine->locked_work_queue is the cross-thread post queue, drained separately after poll. Nothing posted via nxt_event_engine_post() carries an nxt_conn_t today, so the barrier holds, but the wording invited a future reader to assume a conn-bearing post was already covered. The comments now say engine-local and record why the locked queue is outside the barrier. Checking its head here instead was rejected deliberately: nxt_locked_work_queue_t is spinlock-protected, so reading it unlocked would be a data race and locking it would put a spinlock on every engine cycle. Add a debug-only double-free guard. Freeing a conn twice parks it twice, and the promotion walk then splices it onto free_connections twice, leaving the two lists aliased so distinct live connections can be handed the same struct. nxt_conn_is_parked() catches it at the source under NXT_DEBUG. Cover the accounting in the test. test_conn_recycle.py is a body-integrity smoke test that passes identically whether structs are recycled, parked forever, or never pushed -- it cannot see the recycler stop working, which is the failure mode the deferred publish introduces. Assert the /status connection counters over a churn instead, so accepted/closed must match and nothing may be left active or idle. Also forward-declare the file-local helpers at the top, matching the rest of nxt_conn.c. Verified: debug build clean; test_conn_recycle.py both cases pass with the new assertion live. The wider suite could not be re-run here -- a concurrent session holds :8080, which this harness binds during session startup capability checks -- so that re-run is still owed. The accounting test now polls /status until the connection counters settle instead of reading them once. A client-observed EOF is not a server-side barrier: nxt_conn_close() enqueues nxt_conn_shutdown_handler(), which does the SHUT_RDWR the client sees and only then enqueues nxt_conn_close_handler() on the engine's close_work_queue, and nxt_conn_untrack() -- the transition into closed_conns_cnt -- runs in that later handler. So the last connection of a churn could still be counted active while the client had already read the body and closed, failing the test with closed == 59. Reading through to EOF only narrows that window; it does not close it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QgUyzdRHR4e4q6THZZd9Sa --- src/nxt_conn.c | 81 +++++++++++++++++++++++++++++++-------- src/nxt_event_engine.c | 15 ++++++-- test/test_conn_recycle.py | 50 ++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 19 deletions(-) diff --git a/src/nxt_conn.c b/src/nxt_conn.c index 145395cbe..ef10c2103 100644 --- a/src/nxt_conn.c +++ b/src/nxt_conn.c @@ -7,6 +7,13 @@ #include +#if (NXT_DEBUG) +static nxt_bool_t nxt_conn_is_parked(nxt_event_engine_t *engine, + nxt_conn_t *c); +#endif +static nxt_bool_t nxt_conn_timer_quiescent(nxt_timer_t *timer); + + nxt_conn_io_t nxt_unix_conn_io = { .connect = nxt_conn_io_connect, .accept = nxt_conn_io_accept, @@ -153,18 +160,31 @@ nxt_conn_free(nxt_task_t *task, nxt_conn_t *c) nxt_assert(c->task.thread == task->thread); + /* + * Take both timers out of the engine's timer machinery, and take the conn + * off whichever tracking queue it is on. Callers only ever + * nxt_timer_disable() a timer, which clears the enabled bit but leaves the + * node in the engine's rbtree, and they rely on nxt_conn_close_handler() + * having untracked the conn -- both leave a live node inside a struct that + * nxt_conn_create() re-zeroes on reuse, silently corrupting the rbtree or + * the idle/active queue and drifting their counters. nxt_conn_untrack() is + * idempotent and nxt_timer_delete()'s removal can be deferred as a queued + * change, so quiescence is rechecked in nxt_conn_recycle_pending(). + * + * Both run outside the recycling branch below: on the non-recycling path + * the struct dies with mp, so a node left in the rbtree or on a tracking + * queue would dangle into released memory rather than be re-zeroed -- the + * same defect with a worse failure mode. + */ + + nxt_assert(c->idle == NXT_CONN_TRACK_NONE); + nxt_conn_untrack(engine, c); + + nxt_timer_delete(engine, &c->read_timer); + nxt_timer_delete(engine, &c->write_timer); + if (engine->mem_pool != NULL) { - /* - * Take both timers out of the engine's timer machinery. Callers only - * ever nxt_timer_disable() them, which clears the enabled bit but - * leaves the node in the engine's rbtree -- re-zeroing the struct on - * reuse would then corrupt that tree. nxt_timer_delete() removes it, - * but the removal can be deferred as a queued change, so quiescence is - * rechecked in nxt_conn_recycle_pending() before the struct is handed - * out again. - */ - nxt_timer_delete(engine, &c->read_timer); - nxt_timer_delete(engine, &c->write_timer); + nxt_assert(!nxt_conn_is_parked(engine, c)); /* * Park the struct rather than publishing it to the freelist directly. @@ -187,7 +207,7 @@ nxt_conn_free(nxt_task_t *task, nxt_conn_t *c) } -nxt_inline nxt_bool_t +static nxt_bool_t nxt_conn_timer_quiescent(nxt_timer_t *timer) { return !nxt_timer_is_in_tree(timer) @@ -196,12 +216,41 @@ nxt_conn_timer_quiescent(nxt_timer_t *timer) } +#if (NXT_DEBUG) + +/* + * A conn freed twice would appear on pending_connections twice; the promotion + * walk then splices it onto free_connections twice and the two lists end up + * aliasing, so distinct live connections can be handed the same struct. Catch + * it at the source in debug builds rather than at the far-away symptom. + */ + +static nxt_bool_t +nxt_conn_is_parked(nxt_event_engine_t *engine, nxt_conn_t *c) +{ + nxt_conn_t *p; + + for (p = engine->pending_connections; p != NULL; p = p->next) { + if (p == c) { + return 1; + } + } + + return 0; +} + +#endif + + /* * Publish parked connection structs to the reusable freelist. Called from the - * event engine once nxt_event_engine_queue_pop() reports every work queue - * empty: at that point no queued work item can reference any parked struct, so - * reuse cannot race a handler that is still to run. A struct whose timers have - * not settled yet stays parked and is retried on a later cycle. + * event engine once nxt_event_engine_queue_pop() reports every engine-local + * work queue empty: at that point no queued work item can reference any parked + * struct, so reuse cannot race a handler that is still to run. A struct whose + * timers have not settled yet stays parked and is retried on a later cycle. + * + * See nxt_event_engine_start() for why the cross-thread locked_work_queue is + * outside this barrier and why that is currently safe. */ void diff --git a/src/nxt_event_engine.c b/src/nxt_event_engine.c index 6269974fa..079588646 100644 --- a/src/nxt_event_engine.c +++ b/src/nxt_event_engine.c @@ -548,9 +548,18 @@ nxt_event_engine_start(nxt_event_engine_t *engine) if (handler == NULL) { /* - * Every work queue is empty here, so no queued item can still - * reference a connection struct freed during this drain. This - * is the only point at which parked structs become reusable. + * Every engine-local work queue is empty here, so no queued + * item can still reference a connection struct freed during + * this drain. This is the only point at which parked structs + * become reusable. + * + * engine->locked_work_queue is deliberately not covered: it is + * the cross-thread post queue, drained separately by + * nxt_event_engine_post_handler() after poll, and reading its + * head here would race its spinlock. Nothing posted through + * nxt_event_engine_post() carries an nxt_conn_t today, so the + * barrier holds -- but a future post that does would need this + * revisited rather than assumed covered. */ nxt_conn_recycle_pending(engine); break; diff --git a/test/test_conn_recycle.py b/test/test_conn_recycle.py index 1ee826569..3a359bb8e 100644 --- a/test/test_conn_recycle.py +++ b/test/test_conn_recycle.py @@ -1,9 +1,11 @@ import socket +import time from pathlib import Path import pytest from unit.applications.proto import ApplicationProto +from unit.status import Status client = ApplicationProto() @@ -59,6 +61,33 @@ def _churn(n): sock.close() +def _wait_conns(expected, timeout=100): + # A client-observed EOF is not a server-side barrier. nxt_conn_close() + # enqueues nxt_conn_shutdown_handler(), which performs the SHUT_RDWR the + # client sees as EOF and only then enqueues nxt_conn_close_handler() on + # the engine's close_work_queue; nxt_conn_untrack(), which moves a + # connection out of the active/idle queues and into closed_conns_cnt, + # runs in that later handler (src/nxt_conn_close.c). So the last + # connection of a churn can still be counted active after the client has + # read the whole body and closed -- reading through to EOF does not fix + # this, it only narrows the window. + # + # Poll rather than sleep: the close queue drains immediately in the + # common case, while a fixed delay would be dead time on every run and + # still too short under the sanitizer. Return the last reading either + # way, so that a real accounting bug fails as the counter mismatch it is + # rather than as an opaque timeout. + for _ in range(timeout): + conns = Status.get('/connections') + + if conns == expected: + break + + time.sleep(0.1) + + return conns + + def test_conn_recycle_across_thread_churn(): # Populate the per-engine connection freelists. _churn(120) @@ -71,3 +100,24 @@ def test_conn_recycle_across_thread_churn(): _churn(60) assert 'success' in client.conf('4', 'settings/listen_threads') _churn(60) + + +def test_conn_recycle_connection_accounting(): + # The body-integrity churn above passes identically whether structs are + # recycled, parked forever, or never pushed at all, so it cannot see the + # recycler stop working -- notably pending_connections growing without + # bound because some struct never settles. Pin the counters the recycler + # now sits beside instead: every churned connection must be accounted for + # and none may be left behind once the churn is over. + Status.init() + + _churn(60) + + conns = _wait_conns( + {'accepted': 60, 'active': 0, 'idle': 0, 'closed': 60} + ) + + assert conns['accepted'] == 60, 'all accepted' + assert conns['active'] == 0, 'none left active' + assert conns['idle'] == 0, 'none left idle' + assert conns['closed'] == 60, 'all closed' From 4d81e916fb81bb17b1e754140ccb10e02004b446 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 27 Jul 2026 19:08:48 +0200 Subject: [PATCH 004/391] fix(process): take a reference on cross-thread process lookups nxt_runtime_process_find() returns a process without a reference, so a router worker engine could hold the pointer while the router main engine dropped the last reference and ran the teardown -- destroying the shared memory segments, destroying process->incoming.mutex and freeing the process itself. A worker entering nxt_thread_mutex_lock() on that mutex then operated on freed memory. Add nxt_runtime_process_ref(), which does the lookup and the increment in one rt->processes_mutex critical section. Since nxt_process_use() unlinks the process under the same mutex when the count reaches zero, a lookup that still finds the process cannot be racing its teardown. Convert the four call sites that can run on a router worker engine: nxt_port_mmap_handler() released after nxt_port_incoming_port_mmap() returns, inside the success branch -- the fail_close label is shared with the process == NULL path nxt_port_mmap_buf_completion() released inside the oosm block, which the complete_buf back-edge re-enters once per buffer of a chain nxt_port_get_port_incoming_mmap() released strictly after the incoming.mutex unlock, or the last drop would destroy a mutex still held nxt_router_oosm_handler() released once at the end; the process == NULL return precedes the reference nxt_port_remove_pid() is converted too, for a different reason: it runs only on main engines, but the invariant it needs is ownership rather than which thread runs. It hands its result to nxt_process_close_ports(), which takes its own reference around the port loop, so on a process a worker had already dropped to zero that pair is a second 0 -> 1 -> 0 transition and so a second teardown -- racing the one already posted to this engine. Serializing the refcount is what makes that reachable, so it comes in with this change rather than after it. The remaining four lookups stay unreferenced and gain a comment naming the constraint: they run only in the main, prototype and controller processes, each of which has a single event engine. Note nxt_port_mmap_handler() is also in the controller and discovery handler tables, so those processes take the new path too. nxt_process_broadcast_shm_ack() hands a bare process pointer to another engine via nxt_port_post() and the posted handler dereferences it there, which was an independent use-after-free with no refcount at all. The handler now owns a process reference and drops it; the caller compensates only on NXT_ERROR, since nxt_port_post() returns NXT_OK both when it queues the handler and when it calls it inline. That covers the nxt_process_t only: the ports reached through process->ports are separately refcounted and that queue is still walked unlocked on the calling engine, which this does not address. Cost on the shared memory message path is one additional acquisition of rt->processes_mutex per lookup: the find itself took the mutex before this change, so folding the increment into it is free, and only the matching release adds an acquisition. Demonstrated rather than argued. With a delay injected between the lookup and its first use and the peer killed at that point, an ASan build of the parent of this series reports heap-use-after-free on the nxt_process_t in 5 runs out of 5, inside nxt_process_broadcast_shm_ack() reading process->ports on a router worker while the main engine frees the process through nxt_port_remove_pid(). With this commit, the same injection and workload produce none, and the instrumentation shows why: the peer's use_count is 2 rather than 1 for the duration of the window, so the teardown cannot complete. The conversions are not all equally load-bearing, and the injection shows which is. nxt_port_get_port_incoming_mmap() could not be forced open at all: a worker reaching it while reading a response holds a port reference, which holds the process's last reference, so the sleeping worker pins the object it is about to use. The one demonstrably necessary conversion is nxt_port_mmap_buf_completion() -> nxt_process_broadcast_shm_ack(), which runs after that port reference is gone. The other three are converted because the same reasoning does not protect them on every path, not because each was independently reproduced. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ChfpbvjnCiJ2Xohp2S7yLH --- src/nxt_controller.c | 2 ++ src/nxt_main_process.c | 18 ++++++++++++++ src/nxt_port.c | 32 +++++++++++++++++++++++-- src/nxt_port_memory.c | 54 +++++++++++++++++++++++++++++++++++++++--- src/nxt_process.c | 27 ++++++++++++++++++++- src/nxt_router.c | 24 +++++++++++++++++-- src/nxt_runtime.c | 51 +++++++++++++++++++++++++++++++++++++++ src/nxt_runtime.h | 1 + 8 files changed, 201 insertions(+), 8 deletions(-) diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 6c6c691af..c8f418f47 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -495,6 +495,8 @@ nxt_controller_remove_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_memcpy(&pid, msg->buf->mem.pos, sizeof(pid)); + /* Unreferenced: the controller process runs a single engine. */ + process = nxt_runtime_process_find(rt, pid); if (process != NULL && nxt_process_type(process) == NXT_PROCESS_ROUTER) { nxt_controller_router_ready = 0; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index dca79e711..d42b3ef37 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -740,6 +740,14 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; + /* + * Unreferenced: the main process runs a single engine. That matters + * here specifically because the result outlives the call -- it is + * linked into pprocess->children below, a weak link cleaned up in + * nxt_runtime_process_free(). With one engine nothing can drop the + * last reference concurrently, so the link cannot outlive the process. + */ + pprocess = nxt_runtime_process_find(rt, ppid); if (nxt_slow_path(pprocess == NULL)) { nxt_alert(task, "whoami: parent process %PI not found", ppid); @@ -1060,6 +1068,16 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) pid, WEXITSTATUS(status)); } + /* + * Unreferenced: the main process runs a single engine. That + * matters here specifically because nxt_process_close_ports() + * below takes its own +1/-1 around the port loop, which on a + * process already at zero would be a second drop to zero and so a + * second teardown -- the defect fixed in nxt_port_remove_pid(). + * With one engine, find() returning non-NULL implies use_count >= 1 + * for the whole handler. + */ + process = nxt_runtime_process_find(rt, pid); if (process != NULL) { diff --git a/src/nxt_port.c b/src/nxt_port.c index 15a6c59c9..0d2ffe018 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -333,6 +333,8 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; + /* Unreferenced: main and prototype processes run a single engine. */ + process = nxt_runtime_process_find(rt, msg->port_msg.pid); if (nxt_slow_path(process == NULL)) { return; @@ -373,7 +375,15 @@ nxt_port_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - process = nxt_runtime_process_find(rt, msg->port_msg.pid); + /* + * Referenced, not just found: this handler is in the router worker port + * handler table, so it runs on engines other than the one that can free + * the process. nxt_port_incoming_port_mmap() takes and releases + * process->incoming.mutex without touching the reference count, so the + * reference has to span the whole call. + */ + + process = nxt_runtime_process_ref(rt, msg->port_msg.pid); if (nxt_slow_path(process == NULL)) { nxt_log(task, NXT_LOG_WARN, "failed to get process #%PI", msg->port_msg.pid); @@ -383,6 +393,13 @@ nxt_port_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_incoming_port_mmap(task, process, msg->fd[0]); + /* + * Before fail_close: that label is shared with the process == NULL path, + * where no reference was taken. + */ + + nxt_process_use(task, process, -1); + fail_close: nxt_fd_close(msg->fd[0]); @@ -547,10 +564,21 @@ nxt_port_remove_pid(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_rpc_remove_peer(task, msg->port, pid); - process = nxt_runtime_process_find(rt, pid); + /* + * Referenced even though this only runs on main engines: the constraint + * that matters here is ownership, not which thread runs. A router worker + * can drop the last reference to the process at any point, and + * nxt_process_close_ports() takes its own reference around the port loop + * -- on a process already at zero that would be a second drop to zero and + * so a second teardown, racing the one already posted to this engine. + */ + + process = nxt_runtime_process_ref(rt, pid); if (process) { nxt_process_close_ports(task, process); + + nxt_process_use(task, process, -1); } } diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 386407415..5aba9e399 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -230,9 +230,20 @@ nxt_port_mmap_buf_completion(nxt_task_t *task, void *obj, void *data) if (dst_pid == nxt_pid && nxt_atomic_cmp_set(&hdr->oosm, 1, 0)) { - process = nxt_runtime_process_find(task->thread->runtime, src_pid); + process = nxt_runtime_process_ref(task->thread->runtime, src_pid); nxt_process_broadcast_shm_ack(task, process); + + /* + * Released here rather than at the end of the function: the + * complete_buf back-edge below re-enters this block once per buffer + * of the chain, so a release outside it would leak every reference + * but the last. + */ + + if (process != NULL) { + nxt_process_use(task, process, -1); + } } release_buf: @@ -616,7 +627,14 @@ nxt_port_get_port_incoming_mmap(nxt_task_t *task, nxt_pid_t spid, uint32_t id) nxt_process_t *process; nxt_port_mmap_handler_t *mmap_handler; - process = nxt_runtime_process_find(task->thread->runtime, spid); + /* + * Referenced, not just found. This runs on whichever router worker + * engine received the message, while the router main engine can be + * releasing the same process; without the reference the mutex locked on + * the next line can already have been destroyed and freed underneath us. + */ + + process = nxt_runtime_process_ref(task->thread->runtime, spid); if (nxt_slow_path(process == NULL)) { return NULL; } @@ -644,6 +662,17 @@ nxt_port_get_port_incoming_mmap(nxt_task_t *task, nxt_pid_t spid, uint32_t id) nxt_thread_mutex_unlock(&process->incoming.mutex); + /* + * Strictly after the unlock. If this is the last reference, the release + * runs nxt_thread_mutex_destroy(&process->incoming.mutex) -- dropping it + * before the unlock would destroy the mutex this function still holds. + * + * The returned handler is unaffected: the reference bumped above is its + * own, independent of the process, and the caller releases it. + */ + + nxt_process_use(task, process, -1); + return mmap_handler; } @@ -986,7 +1015,24 @@ nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process) port = nxt_process_port_first(process); if (port->type == NXT_PROCESS_APP) { - nxt_port_post(task, port, nxt_port_broadcast_shm_ack, process); + /* + * The process is handed to another engine as a bare pointer and is + * dereferenced there, so the posted handler needs a reference of its + * own and drops it when it is done. nxt_port_post() returns NXT_OK + * both when it queues the handler and when it calls it inline on the + * current engine, and the handler owns the reference either way; only + * the NXT_ERROR case leaves nothing to run, so only that case has to + * be undone here. The inline call is safe: the caller still holds + * its own reference, so this one cannot be the last. + */ + + nxt_process_use(task, process, 1); + + if (nxt_slow_path(nxt_port_post(task, port, nxt_port_broadcast_shm_ack, + process) != NXT_OK)) + { + nxt_process_use(task, process, -1); + } } } @@ -1002,4 +1048,6 @@ nxt_port_broadcast_shm_ack(nxt_task_t *task, nxt_port_t *port, void *data) (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_SHM_ACK, -1, 0, 0, NULL); } nxt_queue_loop; + + nxt_process_use(task, process, -1); } diff --git a/src/nxt_process.c b/src/nxt_process.c index 62c00e6e4..831b7908f 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -347,7 +347,25 @@ nxt_process_child_fixup(nxt_task_t *task, nxt_process_t *process) rt = task->thread->runtime; - /* Remove not ready processes. */ + /* + * Remove not ready processes. + * + * These walks hand unreferenced processes to nxt_process_close_ports(), + * which takes its own +1/-1 -- the shape that made nxt_port_remove_pid() + * a double release once the refcount became cross-thread. It is safe + * here for a different reason than everywhere else, and not because of + * the refcount: this runs immediately after fork() in the child, which + * is single-threaded by definition, so no other thread exists to have + * dropped the last reference. nxt_runtime_process_each() iterating + * rt->processes without the mutex is safe for the same reason. + * + * The other half of that, now that nxt_process_use() locks + * rt->processes_mutex: the mutex is inherited across fork() and would be + * held forever in the child if any other thread of the parent held it at + * the call. None can -- only main and prototype fork, both take the + * mutex on their single event engine, and their thread-pool threads + * never touch it. + */ nxt_runtime_process_each(rt, p) { if (nxt_proc_keep_matrix[ptype][nxt_process_type(p)] == 0 @@ -1291,6 +1309,13 @@ nxt_process_type(nxt_process_t *process) } +/* + * The caller must hold a reference to the process. Closing the ports drops + * the reference each of them holds, so this takes one of its own to survive + * the loop -- but on a process that has already reached zero that same pair + * is a fresh 0 -> 1 -> 0 transition, which runs the teardown a second time. + */ + void nxt_process_close_ports(nxt_task_t *task, nxt_process_t *process) { diff --git a/src/nxt_router.c b/src/nxt_router.c index ba3b1a0e2..8876e7412 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -6000,8 +6000,21 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "oosm in %PI", msg->port_msg.pid); - process = nxt_runtime_process_find(task->thread->runtime, - msg->port_msg.pid); + /* + * Referenced, not just found: .oosm is in both the router main and the + * router worker port handler tables, so this runs on engines other than + * the one that can free the process, and the reference has to span the + * incoming.mutex critical section below. + * + * It has to span the broadcast too, but note what that does and does not + * buy: it keeps the nxt_process_t allocated, and nothing more. The ports + * nxt_process_broadcast_shm_ack() reaches through process->ports are + * refcounted separately and that queue is walked here without a lock, so + * this reference does not make the walk itself safe. + */ + + process = nxt_runtime_process_ref(task->thread->runtime, + msg->port_msg.pid); if (nxt_slow_path(process == NULL)) { return; } @@ -6042,6 +6055,13 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (ack) { nxt_process_broadcast_shm_ack(task, process); } + + /* + * The only release point: the process == NULL return above precedes the + * reference, and the loop breaks and continues but never returns. + */ + + nxt_process_use(task, process, -1); } diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 67efddcf6..abc9a0fe3 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -1727,6 +1727,16 @@ nxt_runtime_process_lhq_pid(nxt_lvlhsh_query_t *lhq, nxt_pid_t *pid) } +/* + * Returns the process without a reference: the pointer is only valid for as + * long as nothing can drop the last reference to it. That holds when the + * caller runs on the single engine that owns the process -- the main, the + * prototype or the controller process, each of which has exactly one event + * engine -- and nowhere else. Any caller that can run on a router worker + * engine must use nxt_runtime_process_ref() instead; the router's main + * engine can be freeing the process concurrently. + */ + nxt_process_t * nxt_runtime_process_find(nxt_runtime_t *rt, nxt_pid_t pid) { @@ -1752,6 +1762,47 @@ nxt_runtime_process_find(nxt_runtime_t *rt, nxt_pid_t pid) } +/* + * Same lookup, but the caller adopts a reference to the result and must drop + * it with nxt_process_use(task, process, -1). The find and the increment + * share one rt->processes_mutex critical section, and nxt_process_use() + * unlinks the process from rt->processes under that same mutex when the count + * reaches zero -- so a lookup that still sees the process in the hash cannot + * be racing its teardown, and the reference it takes cannot be an increment + * on an object that is already being freed. + * + * Cost, since this sits on the shared-memory message path: the find already + * took the mutex, so the increment itself is free; the matching release in + * nxt_process_use() is one additional acquisition of the same uncontended + * global mutex per lookup. + */ + +nxt_process_t * +nxt_runtime_process_ref(nxt_runtime_t *rt, nxt_pid_t pid) +{ + nxt_process_t *process; + nxt_lvlhsh_query_t lhq; + + process = NULL; + + nxt_runtime_process_lhq_pid(&lhq, &pid); + + nxt_thread_mutex_lock(&rt->processes_mutex); + + if (nxt_lvlhsh_find(&rt->processes, &lhq) == NXT_OK) { + process = lhq.value; + process->use_count++; + + } else { + nxt_thread_log_debug("process %PI not found", pid); + } + + nxt_thread_mutex_unlock(&rt->processes_mutex); + + return process; +} + + static nxt_process_t * nxt_runtime_process_get(nxt_runtime_t *rt, nxt_pid_t pid) { diff --git a/src/nxt_runtime.h b/src/nxt_runtime.h index 82ba71bb9..734a23d7d 100644 --- a/src/nxt_runtime.h +++ b/src/nxt_runtime.h @@ -116,6 +116,7 @@ void nxt_runtime_process_unlink_locked(nxt_runtime_t *rt, nxt_process_t *process); nxt_process_t *nxt_runtime_process_find(nxt_runtime_t *rt, nxt_pid_t pid); +nxt_process_t *nxt_runtime_process_ref(nxt_runtime_t *rt, nxt_pid_t pid); nxt_process_t *nxt_runtime_process_first(nxt_runtime_t *rt, nxt_lvlhsh_each_t *lhe); From 14ae9aa21e9771810e64ea438a97ee75bb0602ed Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 1 Aug 2026 02:27:23 +0200 Subject: [PATCH 005/391] test(process): exercise application teardown under shared memory load Drives several router worker engines (settings/listen_threads) serving 1 MiB POSTs to a PHP application while a second thread rewrites that application's configuration, so the router repeatedly builds a new nxt_app_t and tears the previous one down -- destroying its port mmap segments while workers on other engines are still copying request bodies into, and reading responses out of, shared memory for the same application. A 200 response body must be a prefix of the payload: a short body is legitimate when the application dies mid-response, but wrong bytes are what reading recycled shared memory looks like. The test also asserts that more than one application PID served the run, otherwise it silently degrades into a plain concurrency test. A green run is weak evidence for the absence of a use-after-free -- the window is a handful of instructions wide and an uninstrumented build will usually miss it. The value is as a crash and sanitizer driver; the docstring says so rather than overselling it. skip_fds_check(router=True) is needed because destroying a router engine leaks its port socketpair, which is a pre-existing defect in engine teardown unrelated to this workload. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ChfpbvjnCiJ2Xohp2S7yLH --- src/nxt_router.c | 9 +- test/test_process_teardown_churn.py | 236 ++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 test/test_process_teardown_churn.py diff --git a/src/nxt_router.c b/src/nxt_router.c index 8876e7412..5ac3b2748 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -6002,9 +6002,12 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* * Referenced, not just found: .oosm is in both the router main and the - * router worker port handler tables, so this runs on engines other than - * the one that can free the process, and the reference has to span the - * incoming.mutex critical section below. + * router worker port handler tables, and the reference has to span the + * incoming.mutex critical section below. Which engine actually receives + * a given OOSM depends on the port libunit sends it to, so this is not + * assumed either way -- what makes the reference necessary is that a + * worker engine can drop the last reference to the same process while + * this runs. * * It has to span the broadcast too, but note what that does and does not * buy: it keeps the nxt_process_t allocated, and nothing more. The ports diff --git a/test/test_process_teardown_churn.py b/test/test_process_teardown_churn.py new file mode 100644 index 000000000..ea5d72bba --- /dev/null +++ b/test/test_process_teardown_churn.py @@ -0,0 +1,236 @@ +"""Application teardown churn under concurrent shared-memory traffic. + +What this drives: several router worker engines (``settings/listen_threads``) +serving large POSTs to a PHP application while a second thread repeatedly +rewrites that application's configuration. Every configuration rewrite makes +the router build a *new* ``nxt_app_t`` and release the previous one +(``src/nxt_router.c``: an application is reused only when the printed JSON of +its configuration is byte-identical), which tears the old application's +processes down and destroys its outgoing port-mmap segments +(``nxt_port_mmaps_destroy()``) — while router workers on other engines are +still copying request bodies into, and reading responses out of, shared memory +for that same application. + +What this proves and what it does not: a green run is *weak* evidence for the +absence of a use-after-free on the process/port lookup path. The window is a +handful of instructions wide and an unsanitised build will almost always miss +it. The value of this test is as a crash and ASan/UBSan *driver*: run it under +``.github/workflows/sanitize.yml``-style instrumentation and a live +use-after-free becomes a hard failure instead of a coin flip. Read a pass here +as "the workload ran and the daemon stayed up", nothing stronger. + +Deliberately kept short - 64 requests of 1 MiB, about three seconds on an idle +8-core box - so it is cheap enough to run on every CI leg, including the much +slower sanitiser legs. Correctness assertions are written so that a +legitimately failed request during teardown is not a failure, but a corrupt or +hung one is. +""" + +import subprocess +import threading +import time + +from unit.applications.lang.php import ApplicationPHP +from unit.log import Log + +prerequisites = {'modules': {'php': 'all'}} + +client = ApplicationPHP() + +# src/nxt_port_memory_int.h: PORT_MMAP_CHUNK_SIZE is 16 KiB and one shared +# memory segment holds PORT_MMAP_DATA_SIZE (10 MiB) of payload. The router +# always hands requests to an application through those segments +# (nxt_router_prepare_msg() allocates with nxt_port_mmap_get_buf()), but a +# small request fits in a single chunk of a single segment. 1 MiB spans 64 +# chunks in each direction and keeps the worker inside the chunk allocator +# (nxt_port_mmap_get_buf()/nxt_port_mmap_increase_buf()) long enough for a +# teardown on another thread to overlap it. +BODY_SIZE = 1024 * 1024 +BODY = 'x' * BODY_SIZE + +# > 1 so more than one router worker engine is live; this is a top-level +# "settings" member (src/nxt_conf_validation.c: nxt_conf_vldt_setting_members). +LISTEN_THREADS = 4 + +WORKERS = 4 +REQUESTS_PER_WORKER = 16 +CHURN_INTERVAL = 0.05 + +# Not the suite default (60s), so that a stalled request returns instead of +# calling pytest.fail() on a helper thread; the stall is detected by timing it +# and reported as a hard failure below. +READ_TIMEOUT = 10 + +APP = 'mirror' + + +def _app_pids(): + """PIDs currently serving the application, via ps (as conftest does).""" + out = subprocess.check_output( + ['ps', '-ax', '-o', 'pid', '-o', 'command'] + ).decode() + + marker = f'unit: "{APP}" application' + + return {line.split()[0] for line in out.splitlines() if marker in line} + + +def test_process_teardown_churn(skip_fds_check): + # Setting "listen_threads" changes the number of router worker engines, + # and destroying an engine leaks its descriptors: measured on this tree, + # shrinking the count frees each dead engine's epoll and eventfd but keeps + # its port socketpair open (3 sockets per destroyed engine), so the router + # ends the test with more descriptors than it started with. That is a + # pre-existing defect in engine teardown, not something this workload + # causes, and it is unrelated to what the test is here to exercise. + skip_fds_check(router=True) + + client.load(APP, processes={"spare": 2, "max": 4, "idle_timeout": 5}) + + assert 'success' in client.conf( + {"listen_threads": LISTEN_THREADS}, 'settings' + ), 'listen_threads' + + assert client.post(body='ping', read_timeout=READ_TIMEOUT)['status'] == 200 + + lock = threading.Lock() + stop = threading.Event() + failures = [] + outcomes = [] + pids = set() + + def record(bucket, item): + with lock: + bucket.append(item) + + def request_worker(): + for _ in range(REQUESTS_PER_WORKER): + started = time.time() + + try: + resp = client.post( + body=BODY, + read_timeout=READ_TIMEOUT, + read_buffer_size=65536, + ) + + except KeyboardInterrupt: + raise + + # pytest.fail() inside the helpers raises BaseException + except BaseException as exc: # noqa: BLE001 + record(failures, f'request raised {exc!r}') + continue + + elapsed = time.time() - started + status = resp.get('status') + body = resp.get('body', '') + + if status == 200: + # A short body is legitimate: the application can die after its + # headers are on the wire. Wrong *bytes* are not - that is + # what reading freed or recycled shared memory looks like. + if body != BODY[: len(body)]: + # next() needs a default. A body that is all 'x' but + # LONGER than the request still fails the comparison + # above -- BODY[:len(body)] clamps to BODY -- and then + # has no mismatching byte to find. Without the default + # that raises StopIteration here, outside the try that + # guards the request, killing the worker thread instead + # of recording the corruption: the detector fails open + # on one of the shapes it exists to catch. + at = next( + (i for i, c in enumerate(body) if c != 'x'), None + ) + record( + failures, + f'corrupt body: {len(body)} bytes, ' + + ( + f'first mismatch at {at}' + if at is not None + else f'uniform but longer than {BODY_SIZE}' + ), + ) + continue + + full = len(body) == BODY_SIZE + + record(outcomes, 200 if full else 'truncated') + + elif status is None: + # No parsable response: connection closed during teardown. + # Distinguish that from a hang, which is a real defect. + if elapsed >= READ_TIMEOUT: + record(failures, f'no response in {elapsed:.1f}s') + else: + record(outcomes, 'closed') + + elif status in (500, 503): + record(outcomes, status) + + else: + record(failures, f'unexpected status {status}') + + def churn_worker(): + generation = 0 + + while not stop.is_set(): + generation += 1 + + try: + resp = client.conf( + {"CHURN": str(generation)}, + f'applications/{APP}/environment', + ) + + except KeyboardInterrupt: + raise + + except BaseException as exc: # noqa: BLE001 + record(failures, f'config update {generation} raised {exc!r}') + return + + if 'success' not in resp: + record(failures, f'config update {generation} failed: {resp}') + return + + with lock: + pids.update(_app_pids()) + + time.sleep(CHURN_INTERVAL) + + churn = threading.Thread(target=churn_worker) + workers = [threading.Thread(target=request_worker) for _ in range(WORKERS)] + + churn.start() + for worker in workers: + worker.start() + + for worker in workers: + worker.join(timeout=120) + assert not worker.is_alive(), 'request thread hung' + + stop.set() + churn.join(timeout=60) + assert not churn.is_alive(), 'churn thread hung' + + assert not failures, f'{len(failures)} failure(s), first: {failures[0]}' + + # The daemon is still alive and still serves the whole payload correctly. + resp = client.post(body=BODY, read_timeout=READ_TIMEOUT, + read_buffer_size=65536) + + assert resp['status'] == 200, 'still serving' + assert len(resp['body']) == BODY_SIZE, 'still serving whole body' + assert resp['body'] == BODY, 'still serving intact body' + + # The workload has to have actually torn processes down, otherwise it is + # only a concurrency test. More than one PID means the application was + # destroyed and respawned while requests were in flight. + assert len(pids) > 1, f'application processes were not replaced: {pids}' + + # Requests must not have been shut out entirely, or nothing reached shared + # memory concurrently with a teardown. + assert outcomes.count(200) > 0, f'no request completed: {outcomes}' + + Log.check_alerts() From 6472b53f488d8da11f74605ea56b0ab33455f32a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 1 Aug 2026 02:31:23 +0200 Subject: [PATCH 006/391] test(process): cover the OOSM and abrupt-teardown paths The churn test drives only one teardown path -- a configuration rewrite, which stops an application's processes gracefully. Two of the lookups converted in the previous commit are not on it. test_process_shm_oosm reaches nxt_router_oosm_handler() and, through it, nxt_process_broadcast_shm_ack(). An application limited to a single 10 MiB shared memory segment ("limits": {"shm": 1}) is asked for a 24 MiB response while the client stalls before reading, so the response buffers keep the free map full and nxt_unit_mmap_get() has to send OOSM and wait for the ACK. The test asserts on the "oosm in " record the handler logs on entry, and on the application's "oosm: retry", which it can only reach after an NXT_PORT_MSG_SHM_ACK sent by nxt_port_broadcast_shm_ack(). At 24 MiB that is 12-14 OOSM messages per run; 12 MiB produces none, because the socket buffers absorb the overflow. test_process_abrupt_teardown covers nxt_port_remove_pid(), which a graceful teardown does not reach: it kills the application worker with SIGKILL while 1 MiB requests are in flight, six times. It asserts both "exited on signal 9" and the "port remove pid handler" record from nxt_port_remove_pid() itself. Each of those six rounds has to be a teardown of its own, or the test keeps its assertions while quietly losing the coverage they stand for. The round therefore waits for an application pid the run has not killed before, rather than for any pid at all, and then waits for the signal to have actually taken that worker away before counting the round. Both are cheap, and both turn a timing assumption into something checked: a SIGKILLed worker does leave ps promptly -- the marker is the setproctitle string in /proc//cmdline, and a zombie has no cmdline, so ps falls back to "[comm] ", measured at under 6 ms across 200 kills of a CPU-burning process on a fully loaded machine -- but "promptly" is a scheduling property, and these tests exist to be run on loaded sanitizer machines. The closing assertion is that all six pids are distinct, in place of the original "more than one". Both skip loudly when the log carries no [debug] records, since that is where the evidence lives -- they are useful on the sanitize legs, which configure --debug, and honest about proving nothing on the others. Neither demonstrates the absence of a race; they exist to put these paths under a sanitizer at all. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ChfpbvjnCiJ2Xohp2S7yLH Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014og6SESdCDxXzCpR6H9X33 --- test/php/big_response/index.php | 20 ++ test/test_process_abrupt_teardown.py | 316 +++++++++++++++++++++++++++ test/test_process_shm_oosm.py | 155 +++++++++++++ 3 files changed, 491 insertions(+) create mode 100644 test/php/big_response/index.php create mode 100644 test/test_process_abrupt_teardown.py create mode 100644 test/test_process_shm_oosm.py diff --git a/test/php/big_response/index.php b/test/php/big_response/index.php new file mode 100644 index 000000000..6516a0fc1 --- /dev/null +++ b/test/php/big_response/index.php @@ -0,0 +1,20 @@ + mebibytes of 'x' in 64 KiB writes. Each write goes through + * the SAPI's ub_write -> nxt_unit_response_write(), which allocates the bytes + * out of the application's outgoing shared-memory segments, so a response + * larger than the application's "limits"/"shm" budget cannot be in flight all + * at once: the allocator runs the segment's free map dry and Unit's + * application library reports the out-of-shared-memory condition to the + * router (nxt_unit_send_oosm()). + */ +$mb = isset($_GET['mb']) ? (int) $_GET['mb'] : 32; + +header('Content-Type: text/plain'); +header('Content-Length: ' . ($mb * 1024 * 1024)); + +$chunk = str_repeat('x', 65536); + +for ($i = 0; $i < $mb * 16; $i++) { + echo $chunk; +} diff --git a/test/test_process_abrupt_teardown.py b/test/test_process_abrupt_teardown.py new file mode 100644 index 000000000..e31d154d3 --- /dev/null +++ b/test/test_process_abrupt_teardown.py @@ -0,0 +1,316 @@ +"""Abrupt application death while large requests are in flight. + +What this drives: router worker engines on several event engines +(``settings/listen_threads``) moving 1 MiB request and response bodies through +an application's shared memory, while the test repeatedly ``kill -9``s the one +process serving that application. The application is configured with a single +process, so every kill leaves the router with no worker for the application and +forces a respawn, and every kill lands on a process that is mid-transfer. + +Why that matters here: this is the *abrupt* teardown path, and it is a +different one from the graceful, configuration-driven teardown that +test_process_teardown_churn.py drives. A killed worker is noticed by the main +process' SIGCHLD handler, which broadcasts NXT_PORT_MSG_REMOVE_PID; the router +handles it in ``nxt_router_remove_pid_handler()`` -> ``nxt_port_remove_pid()`` +(src/nxt_port.c). That function was converted from ``nxt_runtime_process_find()`` +to ``nxt_runtime_process_ref()``, and it is the site where review found a double +free: it hands its result to ``nxt_process_close_ports()``, which takes its own ++1/-1 around the port loop, so on a process a router worker had already dropped +to zero that pair is a second 0 -> 1 -> 0 transition and so a second teardown, +racing the one already posted. Nothing else in the suite reaches that site with +shared-memory traffic in flight: test_respawn.py kills an idle worker, and the +churn test never kills anything. + +What this proves and what it does not: a green run is *weak* evidence for the +absence of the double free. The window is a handful of instructions wide and +an uninstrumented build will usually miss it. The value of this test is as a +crash and ASan/UBSan *driver* -- run under ``.github/workflows/sanitize.yml`` +instrumentation, a second teardown of the same process becomes a hard failure. +Read a pass as "workers died hard under load, the router noticed each one, and +the daemon kept serving correctly", nothing stronger. + +Correctness assertions are written so that a request legitimately failed by a +kill is not a failure, but a corrupt or hung one is. About two seconds of +workload: six kills, at least 0.3 s apart, against four request threads. Each +round waits for a worker the run has not killed before, and then for the signal +to have actually removed it, so all six are distinct abrupt teardowns rather +than repeat shots at one corpse. +""" + +import subprocess +import threading +import time + +import pytest + +from unit.applications.lang.php import ApplicationPHP +from unit.log import Log + +prerequisites = {'modules': {'php': 'all'}} + +client = ApplicationPHP() + +APP = 'mirror' + +# Same reasoning as test_process_teardown_churn.py: 1 MiB spans 64 chunks of +# shared memory in each direction (PORT_MMAP_CHUNK_SIZE is 16 KiB, +# src/nxt_port_memory_int.h), so a request is inside the chunk allocator, and +# the process/port lookups that reach it, for long enough that a kill on +# another thread lands in the middle of one. +BODY_SIZE = 1024 * 1024 +BODY = 'x' * BODY_SIZE + +# > 1 so more than one router worker engine is live; this is a top-level +# "settings" member (src/nxt_conf_validation.c: nxt_conf_vldt_setting_members). +LISTEN_THREADS = 4 + +WORKERS = 4 +KILLS = 6 +KILL_INTERVAL = 0.3 + +# Not the suite default (60s), so that a stalled request returns instead of +# calling pytest.fail() on a helper thread; the stall is detected by timing it +# and reported as a hard failure below. +READ_TIMEOUT = 10 + + +def _app_pids(): + """PIDs currently serving the application, via ps (as conftest does).""" + out = subprocess.check_output( + ['ps', '-ax', '-o', 'pid', '-o', 'command'] + ).decode() + + marker = f'unit: "{APP}" application' + + return {line.split()[0] for line in out.splitlines() if marker in line} + + +def _wait_for_fresh_app_pids(spent, timeout=5): + """Wait for a worker that is not one already killed, so every kill in the + loop below lands on a live process instead of on the gap left by the + previous one -- or, worse, on the corpse of it. + + Excluding `spent` rather than just waiting for a non-empty set is what + makes each iteration a distinct abrupt teardown. A SIGKILLed worker + leaves `_app_pids()` promptly, because the marker is the setproctitle + string in /proc//cmdline and a zombie has no cmdline, so ps falls + back to "[comm] " -- but "promptly" is a scheduling property, + and this test exists to be run on loaded sanitizer machines. Making the + loop wait for a *new* pid turns that timing assumption into something the + test checks.""" + + deadline = time.time() + timeout + + while time.time() < deadline: + pids = _app_pids() - spent + + if pids: + return pids + + time.sleep(0.05) + + return set() + + +def _wait_for_pids_gone(pids, timeout=5): + """Wait until none of `pids` serves the application any more, i.e. until + the SIGKILL has actually taken a worker away rather than merely being + queued on one.""" + + deadline = time.time() + timeout + + while time.time() < deadline: + if not (_app_pids() & pids): + return True + + time.sleep(0.05) + + return False + + +def test_process_abrupt_teardown(skip_alert, skip_fds_check): + # Killing workers leaves descriptor accounting unsettled in all three + # long-lived processes for as long as the respawn takes, and changing + # "listen_threads" leaks the destroyed engines' port socketpairs on top of + # that (see test_process_teardown_churn.py). Neither is what this test is + # about. + skip_fds_check(main=True, router=True, controller=True) + + # Expected, and in fact asserted on below: this is how the main process + # reports each kill. + skip_alert(r'app process \d+ exited on signal 9') + + # Exactly one process, so a kill always removes the only worker and the + # router has to tear the process down rather than route around it, and so + # every request thread contends on the same incoming.mutex. + client.load(APP, processes={"spare": 1, "max": 1, "idle_timeout": 30}) + + assert 'success' in client.conf( + {"listen_threads": LISTEN_THREADS}, 'settings' + ), 'listen_threads' + + assert client.post(body='ping', read_timeout=READ_TIMEOUT)['status'] == 200 + + stop = threading.Event() + lock = threading.Lock() + failures = [] + outcomes = [] + killed = [] + + def record(bucket, item): + with lock: + bucket.append(item) + + def request_worker(): + while not stop.is_set(): + started = time.time() + + try: + resp = client.post( + body=BODY, + read_timeout=READ_TIMEOUT, + read_buffer_size=65536, + ) + + except KeyboardInterrupt: + raise + + # pytest.fail() inside the helpers raises BaseException + except BaseException as exc: # noqa: BLE001 + # The listener stays open across a worker death, so a refused + # or reset connection is still not expected; record it and let + # the assertions below decide. + record(failures, f'request raised {exc!r}') + continue + + elapsed = time.time() - started + status = resp.get('status') + body = resp.get('body', '') + + if status == 200: + # A short body is legitimate: the worker can die after its + # headers are on the wire. Wrong *bytes* are not - that is + # what reading freed or recycled shared memory looks like. + if body != BODY[: len(body)]: + # next() needs a default. A body that is all 'x' but + # LONGER than the request still fails the comparison + # above -- BODY[:len(body)] clamps to BODY -- and then + # has no mismatching byte to find. Without the default + # that raises StopIteration here, outside the try that + # guards the request, killing the worker thread instead + # of recording the corruption: the detector fails open + # on one of the shapes it exists to catch. + at = next( + (i for i, c in enumerate(body) if c != 'x'), None + ) + record( + failures, + f'corrupt body: {len(body)} bytes, ' + + ( + f'first mismatch at {at}' + if at is not None + else f'uniform but longer than {BODY_SIZE}' + ), + ) + continue + + record(outcomes, 200 if len(body) == BODY_SIZE else 'truncated') + + elif status is None: + # No parsable response: connection closed as the worker died. + # Distinguish that from a hang, which is a real defect. + if elapsed >= READ_TIMEOUT: + record(failures, f'no response in {elapsed:.1f}s') + else: + record(outcomes, 'closed') + + elif status in (500, 503): + record(outcomes, status) + + else: + record(failures, f'unexpected status {status}') + + workers = [threading.Thread(target=request_worker) for _ in range(WORKERS)] + + for worker in workers: + worker.start() + + for round_ in range(KILLS): + pids = _wait_for_fresh_app_pids(set(killed)) + + if not pids: + record( + failures, f'no fresh application worker for kill {round_ + 1}' + ) + break + + subprocess.call(['kill', '-9', *pids]) + + # Only count the round once the signal has demonstrably removed those + # workers. Otherwise a kill that has not landed yet would leave the + # next iteration looking at the same process, and the run would lose + # abrupt-teardown coverage without losing a single assertion. + if not _wait_for_pids_gone(pids): + record(failures, f'kill -9 did not remove {sorted(pids)}') + break + + killed.extend(pids) + + time.sleep(KILL_INTERVAL) + + stop.set() + + for worker in workers: + worker.join(timeout=120) + assert not worker.is_alive(), 'request thread hung' + + assert not failures, f'{len(failures)} failure(s), first: {failures[0]}' + + # The daemon is still alive and still serves the whole payload correctly. + resp = client.post( + body=BODY, read_timeout=READ_TIMEOUT, read_buffer_size=65536 + ) + + assert resp['status'] == 200, 'still serving' + assert len(resp['body']) == BODY_SIZE, 'still serving whole body' + assert resp['body'] == BODY, 'still serving intact body' + + # Requests must not have been shut out entirely, or nothing was in flight + # when a worker died. + assert outcomes.count(200) > 0, f'no request completed: {outcomes[:20]}' + + # Every round killed a worker that had never been killed before, and the + # loop ran all KILLS of them -- so the router really did tear a process + # down and build a new one, repeatedly, and not two or three times with + # the rest of the rounds shooting at the same corpse. The loop enforces + # the first property; assert it anyway, since it is the coverage claim. + assert len(set(killed)) == len(killed), ( + f'a worker was killed twice: {killed}' + ) + assert len(killed) >= KILLS, ( + f'only {len(killed)} worker(s) killed: {killed}' + ) + + log = Log.read() + + # Available at any log level - this is the alert the main process emits + # from its SIGCHLD handler, and the same handler is what broadcasts + # NXT_PORT_MSG_REMOVE_PID to the router. One per kill. + for pid in set(killed): + assert f'app process {pid} exited on signal 9' in log, ( + f'main process did not report the death of {pid}' + ) + + # The direct evidence - "port remove pid handler" is logged by + # nxt_port_remove_pid() itself - is nxt_debug(), compiled out unless Unit + # was configured with --debug. Skip loudly rather than let the test + # degrade into a smoke test that never checks the site it names. + if '[debug]' not in log: + pytest.skip('remove_pid evidence needs a --debug build') + + for pid in set(killed): + assert f'port remove pid {pid} handler' in log, ( + f'nxt_port_remove_pid() did not run for {pid}' + ) + + Log.check_alerts() diff --git a/test/test_process_shm_oosm.py b/test/test_process_shm_oosm.py new file mode 100644 index 000000000..8c5930a4a --- /dev/null +++ b/test/test_process_shm_oosm.py @@ -0,0 +1,155 @@ +"""Router handling of an application that runs out of shared memory. + +What this drives: a PHP application whose shared-memory budget +(``applications//limits/shm``) is one segment, writing a response several +times larger than that segment. Every write goes through +``nxt_unit_response_write()``, which allocates out of the application's +outgoing segments, and those chunks stay busy until the router has written them +to the client and completed the buffers. The client stalls before reading, so +the queued response keeps the segment's free map full while the application +still has output left; ``nxt_unit_mmap_get()`` (src/nxt_unit.c) then reports +the out-of-shared-memory condition to the router and blocks until it is +acknowledged. + +Why that matters here: the OOSM message is the only thing that reaches +``nxt_router_oosm_handler()`` (src/nxt_router.c), and the ACK the application +waits for is the only thing that reaches ``nxt_process_broadcast_shm_ack()`` +(src/nxt_port_memory.c), which the router sends from the ``hdr->oosm`` branch +of ``nxt_port_mmap_buf_completion()`` as it frees the chunks. All three look +the peer process up by pid, and all three were converted from +``nxt_runtime_process_find()``, which returns a process without a reference, to +``nxt_runtime_process_ref()``. Nothing else in this suite sends an OOSM +message, so before this test those conversions had no coverage at all. + +Which engine runs what: ``nxt_unit_send_oosm()`` always sends to +``lib->router_port``, so in practice ``nxt_router_oosm_handler()`` runs on the +router's *main* engine -- confirmed by the thread id in its log record -- even +though ``.oosm`` is in the worker port handler table too. The reference is +load-bearing anyway, because the engine that runs the handler is not the only +one manipulating the process refcount: with ``listen_threads`` above one, the +worker engines take and drop references of their own on every shared-memory +message, and one of those drops can be the last. ``nxt_port_mmap_buf_completion()`` +does run on a worker engine, since that is where the response was written out. + +What this proves and what it does not: it proves the code ran -- the assertions +are on log records emitted from inside those handlers, not on a response that +would look identical either way. It does *not* prove the absence of the +use-after-free they were converted to fix; reaching the handler is necessary +for that race, not sufficient, and an uninstrumented build will usually survive +a live one. Read a pass as "the OOSM/ACK round trip happened and the daemon +stayed correct". The value of the test is as a driver for the sanitiser legs, +where a stale process pointer becomes a hard failure instead of a coin flip. + +Cheap on purpose: one request and one deliberate client stall, about a second +of work once the PHP worker is warm. +""" + +import time + +import pytest + +from unit.applications.lang.php import ApplicationPHP +from unit.log import Log + +prerequisites = {'modules': {'php': 'all'}} + +client = ApplicationPHP() + +APP = 'big_response' + +# One segment is PORT_MMAP_DATA_SIZE == 10 MiB (src/nxt_port_memory_int.h), and +# nxt_unit_init() rounds "limits"/"shm" up to whole segments with a floor of +# one, so any value at or below 10 MiB leaves the application a single segment. +SHM_LIMIT = 1 + +# Comfortably past that 10 MiB even after the kernel socket buffers have +# absorbed their share of the response. Measured on this tree: 12 MiB produces +# no OOSM at all, because the socket buffers alone hold the overflow; 16 MiB +# produces 1-9 messages depending on timing; 24 MiB produced 12-14 on every +# run. Sized for the margin, not for the minimum. +RESPONSE_MB = 24 +RESPONSE_SIZE = RESPONSE_MB * 1024 * 1024 + +# The client sends, then does not read for this long, so the router's queued +# response buffers - which are the application's shared memory, busy until the +# write to the client completes - pile up instead of draining. +STALL = 0.3 + +# > 1 so router worker engines exist and are taking and dropping their own +# references on the same process while the OOSM round trip runs. +LISTEN_THREADS = 4 + +READ_TIMEOUT = 30 + + +def _body(resp, size): + """Trailing `size` bytes of a raw response, i.e. the body: the application + sets Content-Length and the connection is not chunked. Kept as bytes so a + 24 MiB response is not decoded into a second copy.""" + + return resp[-size:] + + +def test_process_shm_oosm(skip_fds_check): + # Same pre-existing engine-teardown descriptor leak that + # test_process_teardown_churn.py documents: changing "listen_threads" + # destroys event engines, and each destroyed engine keeps its port + # socketpair open. Unrelated to what this test exercises. + skip_fds_check(router=True) + + client.load(APP, limits={"shm": SHM_LIMIT}, processes=1) + + assert 'success' in client.conf( + {"listen_threads": LISTEN_THREADS}, 'settings' + ), 'listen_threads' + + sock = client.get(url=f'/?mb={RESPONSE_MB}', no_recv=True) + + time.sleep(STALL) + + resp = client.recvall(sock, read_timeout=READ_TIMEOUT, buff_size=65536) + sock.close() + + body = _body(resp, RESPONSE_SIZE) + + assert len(body) == RESPONSE_SIZE, 'whole response' + assert body.count(b'x') == RESPONSE_SIZE, 'intact response' + + log = Log.read() + + # The records asserted on below are nxt_debug()/nxt_unit_debug(), compiled + # out unless Unit was configured with --debug. Skip loudly rather than + # let the test degrade into a smoke test that proves nothing about the + # path it claims to cover. The sanitiser workflow builds --debug, which + # is the leg this test exists for. + if '[debug]' not in log: + pytest.skip('OOSM evidence needs a --debug build') + + # nxt_router_oosm_handler() logs this on entry, immediately before the + # reference it now takes; its presence is the proof that the converted + # call site ran. + assert 'oosm in ' in log, 'router handled an OOSM message' + + # The application logs "oosm: retry" only after nxt_unit_wait_shm_ack() + # has returned OK, and that happens only on an NXT_PORT_MSG_SHM_ACK + # message (src/nxt_unit.c). The router's only sender of that message is + # nxt_port_broadcast_shm_ack(), reached through the converted + # nxt_process_broadcast_shm_ack() -- so this record is proof that the + # cross-engine post, and the reference it now carries, ran too. + # + # Which of the two callers broadcast is deliberately not asserted: with + # the segment this full, nxt_router_oosm_handler() almost never finds a + # free chunk to acknowledge with ("oosm: already free" is absent from + # these runs) and the ACK comes from nxt_port_mmap_buf_completion(). + assert 'oosm: retry' in log, 'application got a SHM_ACK back' + + # Still serving, from the same worker, after the round trip. + sock = client.get(url='/?mb=1', no_recv=True) + resp = client.recvall(sock, read_timeout=READ_TIMEOUT, buff_size=65536) + sock.close() + + assert _body(resp, 1024 * 1024).count(b'x') == 1024 * 1024, ( + 'still serving' + ) + + Log.check_alerts() From ca47160b65d75202bd679e15b53c9eb4ca9ef1da Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 27 Jul 2026 18:16:34 +0200 Subject: [PATCH 007/391] port: drop the dead tracking bitmap from the shm segment header nxt_port_mmap_header_t is mapped over the first PORT_MMAP_HEADER_SIZE (4096) bytes of a shared segment, with chunk 0 starting immediately after. The struct was 5320 bytes. Its last three fields -- free_tracking_map[10], free_tracking_map_padding and tracking[640] -- ran from offset 112 to 5320, so tracking[] aliased the first 1224 bytes of chunk 0's payload. That is harmless today only because the field has no readers and no writers anywhere in the tree: the chunk-tracking mechanism that used it was removed upstream in b4540f09, leaving the storage behind. The day anyone reintroduces tracking, the first write silently corrupts the payload of whatever request is holding chunk 0. Delete the three fields. free_tracking_map survived only as the "true" arm of two `tracking ? hdr->free_tracking_map : hdr->free_map` selectors in nxt_port_mmap_get() and nxt_port_new_port_mmap(); the sole caller, nxt_port_mmap_get_buf(), passes a literal 0, so the parameter and both selectors go with it, along with the initialisations in nxt_port_new_port_mmap() and nxt_unit_new_mmap(). This is a shared-memory struct read by two processes that need not be running the same build -- libunit is linked into application modules -- so the layout change has to be argued, not assumed: - All three fields are trailing. id (0), src_pid (4), dst_pid (8), sent_over (12), oosm (16), free_map (24) and free_map_padding (104) keep their offsets exactly; sizeof() drops 5320 -> 112. Every field a peer of either vintage actually reads or writes is untouched. - The mapping length is PORT_MMAP_SIZE, never sizeof(header), so the segment size and the address of chunk 0 (hdr + 4096) are unchanged. - An old peer initialising free_tracking_map writes [112, 200) and its sentinel word at [192, 200). Both now land in the unused space between the end of the struct and the 4096-byte header boundary, which no build reads. An old peer never touches tracking[] at all, for the same reason a new one does not: b4540f09 removed the code. free_map_padding stays. It is not alignment padding: with PORT_MMAP_CHUNK_COUNT = 640 and 64 chunks per word, free_map[] covers indices 0..9 and nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT) deliberately writes free_map[10] -- the padding word -- to plant a permanently-busy sentinel there. It is what makes a multi-chunk allocation starting near the end of the segment fail its nxt_port_mmap_chk_set_chunk_busy() continuation instead of running off the array. Removing it would turn a by-design write into an out-of-bounds one, so the field keeps its storage and gains a comment saying why. Add nxt_static_assert() to nxt_clang.h and assert the invariant that was being violated: sizeof(nxt_port_mmap_header_t) <= PORT_MMAP_HEADER_SIZE. The tree builds as gnu11, but libunit headers are also compiled by whatever toolchain an application module uses, so the macro resolves to _Static_assert only under __STDC_VERSION__ >= 201112L and falls back to a negative-sized array typedef otherwise, following the same #if/#else shape the rest of nxt_clang.h uses for compiler-feature differences. Verified that the assertion can fail rather than merely compile: adding a char[PORT_MMAP_HEADER_SIZE] probe to the struct produces src/nxt_clang.h:28:5: error: static assertion failed: "nxt_port_mmap_header_t overflows the segment header area" src/nxt_port_memory_int.h:89:1: note: in expansion of macro 'nxt_static_assert' and the pre-C11 arm was exercised separately (gcc/clang -std=gnu89 and -std=gnu99: "size of array 'nxt_static_assert_failed_3' is negative"), with the passing form accepted by gcc and clang under gnu89/gnu99/gnu11/ c2x and by g++ under c++11/c++17. Both geometries fit with room to spare: 112 of 4096 here, 40 of 1024 under NXT_MMAP_TINY_CHUNK. Build is clean under -Wall -Wextra -Werror; build/tests and test_php_application.py (49 passed, 3 skipped) pass. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01R16e64QnGE7MtvQWZyT6wN --- src/nxt_clang.h | 29 +++++++++++++++++++++++++++++ src/nxt_port_memory.c | 18 ++++++------------ src/nxt_port_memory_int.h | 36 +++++++++++++++++++++++++++++++++--- src/nxt_unit.c | 2 -- 4 files changed, 68 insertions(+), 17 deletions(-) diff --git a/src/nxt_clang.h b/src/nxt_clang.h index 872f15271..2453bff31 100644 --- a/src/nxt_clang.h +++ b/src/nxt_clang.h @@ -13,6 +13,35 @@ #define nxt_cdecl +/* + * nxt_static_assert() fails the compilation when the condition does not + * hold. auto/cc/test adds -std=gnu11 only on its gcc and clang branches + * and leaves the generic "cc" branch alone, so a compiler defaulting to + * pre-C11 has to fall back to declaring an array with a negative size; + * there the message survives only in the type name, which has to be unique + * per use, hence the token pasting. + */ + +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) + +#define nxt_static_assert(cond, msg) \ + _Static_assert(cond, msg) + +#else + +#define nxt_static_assert_paste(prefix, line) \ + prefix##line + +#define nxt_static_assert_name(prefix, line) \ + nxt_static_assert_paste(prefix, line) + +#define nxt_static_assert(cond, msg) \ + typedef char nxt_static_assert_name(nxt_static_assert_failed_, \ + __LINE__)[(cond) ? 1 : -1] + +#endif + + #if (NXT_CLANG) /* Any __asm__ directive disables loop vectorization in GCC and Clang. */ diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 5aba9e399..1808bddb7 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -390,13 +390,11 @@ nxt_port_incoming_port_mmap(nxt_task_t *task, nxt_process_t *process, static nxt_port_mmap_handler_t * -nxt_port_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, - nxt_bool_t tracking, nxt_int_t n) +nxt_port_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_int_t n) { void *mem; nxt_fd_t fd; nxt_int_t i; - nxt_free_map_t *free_map; nxt_port_mmap_t *port_mmap; nxt_port_mmap_header_t *hdr; nxt_port_mmap_handler_t *mmap_handler; @@ -438,22 +436,18 @@ nxt_port_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, hdr = mmap_handler->hdr; nxt_memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map)); - nxt_memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map)); hdr->id = mmaps->size - 1; hdr->src_pid = nxt_pid; hdr->sent_over = 0xFFFFu; /* Mark first chunk as busy */ - free_map = tracking ? hdr->free_tracking_map : hdr->free_map; - for (i = 0; i < n; i++) { - nxt_port_mmap_set_chunk_busy(free_map, i); + nxt_port_mmap_set_chunk_busy(hdr->free_map, i); } /* Mark as busy chunk followed the last available chunk. */ nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); - nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT); nxt_log(task, NXT_LOG_DEBUG, "new mmap #%D created for %PI -> ...", hdr->id, nxt_pid); @@ -547,7 +541,7 @@ nxt_shm_open(nxt_task_t *task, size_t size) static nxt_port_mmap_handler_t * nxt_port_mmap_get(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_chunk_id_t *c, - nxt_int_t n, nxt_bool_t tracking) + nxt_int_t n) { nxt_int_t i, res, nchunks; nxt_free_map_t *free_map; @@ -577,7 +571,7 @@ nxt_port_mmap_get(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_chunk_id_t *c, *c = 0; - free_map = tracking ? hdr->free_tracking_map : hdr->free_map; + free_map = hdr->free_map; while (nxt_port_mmap_get_free_chunk(free_map, c)) { nchunks = 1; @@ -611,7 +605,7 @@ nxt_port_mmap_get(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_chunk_id_t *c, end: *c = 0; - mmap_handler = nxt_port_new_port_mmap(task, mmaps, tracking, n); + mmap_handler = nxt_port_new_port_mmap(task, mmaps, n); unlock_return: @@ -705,7 +699,7 @@ nxt_port_mmap_get_buf(nxt_task_t *task, nxt_port_mmaps_t *mmaps, size_t size) b->completion_handler = nxt_port_mmap_buf_completion; nxt_buf_set_port_mmap(b); - mmap_handler = nxt_port_mmap_get(task, mmaps, &c, nchunks, 0); + mmap_handler = nxt_port_mmap_get(task, mmaps, &c, nchunks); if (nxt_slow_path(mmap_handler == NULL)) { mp = task->thread->engine->mem_pool; nxt_mp_free(mp, b); diff --git a/src/nxt_port_memory_int.h b/src/nxt_port_memory_int.h index 63ac09e19..d7fd4e083 100644 --- a/src/nxt_port_memory_int.h +++ b/src/nxt_port_memory_int.h @@ -10,6 +10,7 @@ #include #include +#include #ifdef NXT_MMAP_TINY_CHUNK @@ -70,13 +71,42 @@ struct nxt_port_mmap_header_s { nxt_port_id_t sent_over; nxt_atomic_t oosm; nxt_free_map_t free_map[MAX_FREE_IDX]; + /* + * Not padding in the alignment sense: nxt_port_mmap_set_chunk_busy() is + * called with PORT_MMAP_CHUNK_COUNT to plant a permanently-busy sentinel + * one word past the last real word of free_map[], so that a multi-chunk + * allocation walking off the end of the segment fails to claim its + * continuation instead of reading past the end of the struct. + */ nxt_free_map_t free_map_padding; - nxt_free_map_t free_tracking_map[MAX_FREE_IDX]; - nxt_free_map_t free_tracking_map_padding; - nxt_atomic_t tracking[PORT_MMAP_CHUNK_COUNT]; + + /* + * Not a live field. It reserves the window that a peer built before the + * tracking bitmap was dropped still writes: such a peer memsets + * MAX_FREE_IDX words of free_tracking_map starting here and plants its + * sentinel one word past them. A libunit of that vintage can create + * segments this build maps, so the window has to stay unused while those + * peers are supported. + * + * Reserved in the struct rather than only described in a comment, so + * that the next field added lands after it by construction instead of + * by the author having read this. Removing the reservation is what + * turns the removal of the tracking bitmap into the same latent + * corruption it was meant to fix. + */ + nxt_free_map_t legacy_tracking_window[MAX_FREE_IDX + 1]; }; +/* + * The header struct is mapped over the first PORT_MMAP_HEADER_SIZE bytes of + * the segment and chunk 0 starts right after it, so anything the struct + * declares beyond that boundary silently aliases payload. + */ +nxt_static_assert(sizeof(nxt_port_mmap_header_t) <= PORT_MMAP_HEADER_SIZE, + "nxt_port_mmap_header_t overflows the segment header area"); + + struct nxt_port_mmap_handler_s { nxt_port_mmap_header_t *hdr; nxt_atomic_t use_count; diff --git a/src/nxt_unit.c b/src/nxt_unit.c index dd33408b0..92ed26e15 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -4016,7 +4016,6 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int n) hdr = mem; memset(hdr->free_map, 0xFFU, sizeof(hdr->free_map)); - memset(hdr->free_tracking_map, 0xFFU, sizeof(hdr->free_tracking_map)); hdr->id = lib->outgoing.size - 1; hdr->src_pid = lib->pid; @@ -4031,7 +4030,6 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int n) /* Mark as busy chunk followed the last available chunk. */ nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); - nxt_port_mmap_set_chunk_busy(hdr->free_tracking_map, PORT_MMAP_CHUNK_COUNT); pthread_mutex_unlock(&lib->outgoing.mutex); From 9fcf5b93519e6e04a38af2a7e315dff1f7d9ac03 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 24 Jul 2026 13:38:12 +0200 Subject: [PATCH 008/391] test: exercise router recovery after send-timeout aborts test_settings_send_timeout checks the framing of a single aborted response, but nothing asserted that the router is still usable afterwards. A request that fails to complete on the abort path -- its last buffer completed twice, or never completed at all -- would surface only indirectly, as a teardown failure or a later unrelated test. Abort three streamed responses in a row over a unix listener, then serve a normal request on the same listener and check its body. Unix is the shape that reproduced the abort-path completion bugs most reliably; test_conn_task_race.py covers the TCP straddle with a hard RST. Give the client socket a timeout: an unresponsive router is precisely what this test looks for, and an unbounded read would hang the run instead of failing it. Drop send_timeout in a finally so the shortened value does not leak into later tests in the no-restart suite. Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ADLFQU9cw2E1EKeJgLDB21 --- test/test_settings.py | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/test/test_settings.py b/test/test_settings.py index 9d37d6cae..64e86150b 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -309,6 +309,72 @@ def req(addr, data_len): assert len(data) > data_len, 'send timeout data 2' +def test_settings_send_timeout_recovery(temp_dir): + # test_settings_send_timeout checks the framing of a single aborted + # response. Nothing asserted that the router is still usable afterwards, + # so a request that fails to complete on the abort path -- e.g. its last + # buffer completed twice, or never -- would surface only indirectly. Abort + # several streamed responses in a row over a unix listener, then serve a + # normal request on the same listener. + client.load('body_generate') + + def req(addr, data_len): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + # A router that stops serving after an abort is exactly what this test + # looks for; without a timeout that failure would hang the run instead + # of failing it. + sock.settimeout(30) + sock.connect(addr) + + req = f"""GET / HTTP/1.1 +Host: localhost +X-Length: {data_len} +Connection: close + +""" + + sock.sendall(req.encode()) + + data = sock.recv(16).decode() + + time.sleep(3) + + data += client.recvall(sock).decode() + + sock.close() + + return data + + sysctl_out = sysctl() + values = re.findall(r'net.core.[rw]mem_(?:max|default).*?(\d+)', sysctl_out) + values = [int(v) for v in values] + + data_len = 1048576 if len(values) == 0 else 10 * max(values) + + addr = f'{temp_dir}/sock' + + assert 'success' in client.conf( + {f'unix:{addr}': {'application': 'body_generate'}}, 'listeners' + ) + + assert 'success' in client.conf({'http': {'send_timeout': 1}}, 'settings') + + try: + for i in range(3): + data = req(addr, data_len) + assert re.search(r'200 OK', data), f'aborted status {i}' + assert len(data) < data_len, f'aborted data {i}' + + data = req(addr, 10) + assert re.search(r'200 OK', data), 'recovery status' + assert re.search(r'XXXXXXXXXX$', data), 'recovery data' + + finally: + # The default no-restart suite preserves /settings between tests. + client.conf_delete('settings/http/send_timeout') + + def test_settings_idle_timeout(): client.load('empty') From f3c79f8866ac7e819573c96f1348ed2b938e9fb1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 18 Aug 2026 03:54:37 +0200 Subject: [PATCH 009/391] ci(sanitize): gate on what the logs contain, not on an empty directory The sanitizer job could not fail. Its guard step listed asan-logs/, that directory was always empty, and so every run took the clean branch and printed "No sanitizer reports; lifecycle paths are clean." ASAN_OPTIONS/UBSAN_OPTIONS log_path does not work for unitd. The daemon relocates argv[]/environ[] in nxt_process_title() and overwrites the original block, so the options string the sanitizer runtime kept a pointer into is gone by the time a forked router or application worker writes a report. Measured on this tree: an ordinary binary run with the same UBSAN_OPTIONS honours log_path, halt_on_error and print_stacktrace, while inside unitd none of the three take effect and the log directory stays empty on every run. What that cost, measured on master (02263309) with the configuration-churn workload from #184 under an ASan+UBSan build: - 24 runs, a UBSan violation in every one of them, pytest green in 23. Log.check_alerts() greps unit.log for "[alert]" and "Sanitizer", not for UBSan's "runtime error:", so a UBSan-only violation fails nothing and appears nowhere. - The one run that hit the #186 heap-use-after-free in nxt_locked_work_queue_move() failed only as 35 "Client can't connect to the server" errors -- an infrastructure flake, to any reader -- while the gate still reported the leg clean. Reports do reach unitd's stderr, which conftest.py wires to /unit.log. So: drop log_path, pass --save-log so conftest keeps that file instead of removing the temp dir after each test, and scan it. The scan lives in a script rather than inline YAML so it can be run against a captured log, which is how each of its five behaviours was checked: a leg whose only violations are allowlisted passes; an unknown UBSan site fails; the real #186 ASan log fails; an ASan report stays failing even against an allowlist containing ".*"; and a run that produced no unit.log at all is a hard error rather than a pass, since "no evidence" must not read as "no findings". grep -a is load-bearing there. The --debug log carries raw payload bytes -- tens of thousands of NULs in a typical run -- so grep classifies unit.log as binary, prints the matches preceding the first NUL and stops. On the captured churn log that dropped 5 of 8 report lines, including both AddressSanitizer lines. Without -a this gate reproduces the bug it removes. Pre-existing UBSan sites are waived in .github/sanitizer-allowlist.txt rather than fixed here, so that the gate becomes blocking in one step instead of waiting on unrelated fixes. Entries pin a source location and name the issue tracking the fix; ASan/LSan/TSan reports are not waivable there by construction. Fixes #185 --save-log needs a matching reset of the harness's log read offsets, or it trades one silent false-green for another. Teardown records the offset it read to unconditionally and only resets it when it removes the temp dir, which --save-log is precisely what stops it from doing. Under --restart every test also gets a FRESH temp dir and an empty unit.log, so the offset saved against the previous test's file was being applied to the next one: Log.open() seeked past the end of a smaller log and every log-based read returned nothing. Caught by CI, where test_sigterm_drops_inflight_request failed asserting on a record that was present in its log at byte 215980 while the offset carried in from the previous test was 254889. The visible failure was one assertion; the quiet part was worse. The same offset feeds the teardown Log.check_alerts() and _check_fds(), so under --save-log --restart those stopped examining most of what they were handed -- the exact shape of defect this commit exists to remove, reintroduced one layer down. unit_run() now clears Log.pos when it creates the temp dir, which covers swapped log names and the session-start run as well as the per-test restart. The path filters name this workflow but not the two files the gate step now depends on, so a change confined to the scan script or the waiver list would skip the job that exists to validate it -- a change that broke report detection, or that widened what is ignored, would arrive unchecked. Both are added to the pull_request and push filters. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014og6SESdCDxXzCpR6H9X33 --- .github/sanitizer-allowlist.txt | 32 ++++++ .github/scripts/check-sanitizer-reports.sh | 126 +++++++++++++++++++++ .github/workflows/sanitize.yml | 87 +++++++------- test/conftest.py | 11 ++ 4 files changed, 214 insertions(+), 42 deletions(-) create mode 100644 .github/sanitizer-allowlist.txt create mode 100755 .github/scripts/check-sanitizer-reports.sh diff --git a/.github/sanitizer-allowlist.txt b/.github/sanitizer-allowlist.txt new file mode 100644 index 000000000..b09bddddd --- /dev/null +++ b/.github/sanitizer-allowlist.txt @@ -0,0 +1,32 @@ +# Known UndefinedBehaviorSanitizer violations, waived by +# .github/scripts/check-sanitizer-reports.sh so that the sanitizer gate can be +# made blocking without first fixing every pre-existing site. +# +# Rules: +# - One POSIX extended regular expression per line, matched against the +# report line as it appears in the harness's unit.log. +# - Every entry names the issue tracking its fix. An entry without one is a +# bug in this file: the point of the list is to shrink, and an unattributed +# waiver never does. +# - ASan/LSan/TSan reports cannot be waived here. The script refuses to +# match them on purpose -- a memory-safety report fails the leg, always. +# - Pin the source location. A bare message would waive the same class of +# violation everywhere in the tree, including a new one. +# +# Measured on master (02263309) with the php leg's own test set: entries 1 and +# 2 fire on every run of test/test_php_application.py, entry 3 on the +# configuration-churn workload. + +# freeunitorg/freeunit#188 -- nxt_conf_value_t union read through an address +# the allocator only aligned to 4 bytes. Benign on x86-64, a real fault on +# strict-alignment targets; tracked for a fix in the value representation. +src/nxt_conf\.c:[0-9]+:[0-9]+: runtime error: member access within misaligned address 0x[0-9a-f]+ for type 'union ', which requires 8 byte alignment + +# freeunitorg/freeunit#200 -- nxt_strstr_eq() -> memcmp() on a zero-length +# nxt_str_t, whose .start is NULL. memcmp() declares both pointers nonnull +# regardless of length, so a zero-length compare is UB even though every +# implementation returns 0 without a dereference. +src/nxt_http_route\.c:[0-9]+:[0-9]+: runtime error: null pointer passed as argument [12], which is declared to never be null + +# freeunitorg/freeunit#200 -- same class in nxt_cpymem(): memcpy(dst, NULL, 0). +src/nxt_string\.h:[0-9]+:[0-9]+: runtime error: null pointer passed as argument 2, which is declared to never be null diff --git a/.github/scripts/check-sanitizer-reports.sh b/.github/scripts/check-sanitizer-reports.sh new file mode 100755 index 000000000..63542cbba --- /dev/null +++ b/.github/scripts/check-sanitizer-reports.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# +# Fail a sanitizer CI leg on any ASan/UBSan report the test session produced. +# +# Why this scans the harness's unit.log rather than a sanitizer log_path +# directory: ASAN_OPTIONS/UBSAN_OPTIONS log_path does not work for unitd. +# The daemon relocates argv[]/environ[] in nxt_process_title() +# (src/nxt_process_title.c) and overwrites the original block, so by the time a +# forked router or application worker writes a report the options string the +# sanitizer runtime kept a pointer into is gone. Measured on this tree: an +# ordinary binary run with the same UBSAN_OPTIONS honours log_path, +# halt_on_error and print_stacktrace; inside unitd none of the three take +# effect and the log directory stays empty on every run -- including runs that +# produced a genuine heap-use-after-free. Reports do reach unitd's stderr, +# which test/conftest.py wires to /unit.log, so that is what we read. +# +# The caller must pass --save-log to pytest: without it conftest.py removes the +# per-restart temp dir (and with it unit.log) after every test. +# +# Usage: check-sanitizer-reports.sh [allowlist-file] +# directory holding the harness's unit-test-* temp dirs +# (tempfile.mkdtemp(prefix='unit-test-'), i.e. $TMPDIR or /tmp) +# [allowlist-file] optional; see .github/sanitizer-allowlist.txt + +set -eu + +log_root=${1:?usage: check-sanitizer-reports.sh [allowlist-file]} +allowlist=${2:-} + +# A sanitizer report line, anchored tightly enough that a request body echoed +# into the --debug log cannot forge one. +# UBSan: "src/nxt_conf.c:664:26: runtime error: ..." +# ASan: "==123==ERROR: AddressSanitizer: heap-use-after-free ..." +ubsan_re='[^[:space:]]+:[0-9]+:[0-9]+: runtime error: ' +asan_re='(ERROR|SUMMARY): [A-Za-z]*Sanitizer:' + +logs=$(find "$log_root" -maxdepth 2 -name unit.log -type f 2>/dev/null | sort) + +if [ -z "$logs" ]; then + echo "::error::No unit.log found under ${log_root}. The sanitizer gate" \ + "cannot certify this leg: either the test session never started, or" \ + "pytest was invoked without --save-log and the harness removed the" \ + "temp dirs. Refusing to report a clean run." + exit 1 +fi + +echo "Scanning $(echo "$logs" | wc -l) unit.log file(s) under ${log_root}:" +echo "$logs" | sed 's/^/ /' +echo + +findings=$(mktemp) +patterns=$(mktemp) +trap 'rm -f "$findings" "$patterns"' EXIT + +# -a is load-bearing, not defensive. unitd's --debug log carries raw payload +# bytes, tens of thousands of NULs in a typical run, so grep classifies +# unit.log as binary: it prints the matches it found before the first NUL and +# then silently stops. Measured on a churn log: 3 of 8 report lines survived, +# and the two AddressSanitizer lines -- which come later in the file than the +# first NUL -- were among the 5 dropped. Without -a this gate reproduces the +# exact bug it was written to remove. +for f in $logs; do + grep -aEH "$ubsan_re|$asan_re" "$f" 2>/dev/null || true +done > "$findings" + +if [ ! -s "$findings" ]; then + echo "No sanitizer reports." + exit 0 +fi + +# An ASan/LSan/TSan report is never allowlistable -- those are memory-safety +# bugs, not style debt. Only UBSan lines can be waived, and only by an entry +# that names an open issue. +hard=$(grep -aE "$asan_re" "$findings" || true) + +soft=$(grep -aEv "$asan_re" "$findings" || true) +known='' +unknown=$soft + +if [ -n "$allowlist" ] && [ -f "$allowlist" ] && [ -n "$soft" ]; then + # Patterns go through a file, not a heredoc: `grep -f -` would take its + # pattern list from the same stdin the findings are piped in on, match + # nothing, and report a clean leg -- the exact failure this gate exists to + # remove. + grep -Ev '^[[:space:]]*(#|$)' "$allowlist" > "$patterns" || true + + if [ -s "$patterns" ]; then + known=$(printf '%s\n' "$soft" | grep -aE -f "$patterns" || true) + unknown=$(printf '%s\n' "$soft" | grep -aE -v -f "$patterns" || true) + fi +fi + +if [ -n "$known" ]; then + echo "Known UBSan violations waived by ${allowlist} (tracked, not fixed here):" + printf '%s\n' "$known" | sed 's/.*unit\.log://' | sort | uniq -c | sort -rn | + sed 's/^/ /' + echo +fi + +if [ -z "$hard" ] && [ -z "$unknown" ]; then + echo "No unwaived sanitizer reports." + exit 0 +fi + +if [ -n "$hard" ]; then + echo "AddressSanitizer/LeakSanitizer report(s) -- never waivable:" + printf '%s\n' "$hard" | sed 's/^/ /' + echo + echo "Stack frames from the same log(s):" + for f in $logs; do + grep -aE '^ *#[0-9]+ 0x' "$f" 2>/dev/null | head -40 | sed 's/^/ /' + done + echo +fi + +if [ -n "$unknown" ]; then + echo "UBSan violation(s) not in ${allowlist:-(no allowlist)}:" + printf '%s\n' "$unknown" | sed 's/.*unit\.log://' | sort | uniq -c | sort -rn | + sed 's/^/ /' + echo +fi + +echo "::error::Sanitizer report(s) in this leg; see the step output above and" \ + "the uploaded unit-logs artifact. A new UBSan site must be fixed, or" \ + "waived in .github/sanitizer-allowlist.txt with an issue reference." +exit 1 diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 7d7bee412..eb19570d2 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -23,7 +23,13 @@ on: - 'go/**' - 'src/**' - 'test/**' + # The gate is three files, not one: a change to the scan script or to + # the waiver list can break report detection or widen what is ignored + # just as effectively as a change to this workflow, and must not skip + # the job that would catch it. - '.github/workflows/sanitize.yml' + - '.github/scripts/check-sanitizer-reports.sh' + - '.github/sanitizer-allowlist.txt' push: # master is the release/default branch; pre-* are the release-prep # integration branches (e.g. pre-1.35.6) where the hardening actually @@ -39,7 +45,13 @@ on: - 'go/**' - 'src/**' - 'test/**' + # The gate is three files, not one: a change to the scan script or to + # the waiver list can break report detection or widen what is ignored + # just as effectively as a change to this workflow, and must not skip + # the job that would catch it. - '.github/workflows/sanitize.yml' + - '.github/scripts/check-sanitizer-reports.sh' + - '.github/sanitizer-allowlist.txt' jobs: sanitize: @@ -176,61 +188,52 @@ jobs: - name: Make ${{ matrix.runtime }} module run: make -j$(nproc) ${{ matrix.make }} - - name: Create ASan log directory - run: mkdir -p "${GITHUB_WORKSPACE}/asan-logs" - # ASAN_OPTIONS / UBSAN_OPTIONS are step-level env so pytest and every # forked unitd process inherit them. - # log_path=/asan-logs/{asan,ubsan} - # ASan and UBSan reports from the forked daemons land in - # asan-logs/. files rather than only the per-test unit.log - # the harness rotates away, so the guard step below can fail the job - # even when pytest itself is green. UBSan needs its OWN log_path: - # with only ASAN_OPTIONS set, a UBSan-only violation that does not - # escalate to a fatal signal writes to neither the asan-logs dir nor - # a place the harness scans (it greps unit.log for "Sanitizer", not - # UBSan's "runtime error"), so half the gate could pass silently. # detect_leaks=0 # the fork-heavy daemon makes LeakSanitizer extremely noisy; turning # leak detection on is tracked as a follow-up, once the teardown # paths are use-after-free / null-deref clean. - # pytest_opts is per-runtime (e.g. --restart for the graceful tests). + # print_stacktrace / halt_on_error + # honoured wherever the options string survives. They do NOT take + # effect inside unitd itself -- see the header of + # .github/scripts/check-sanitizer-reports.sh -- which is exactly why + # the gate below reads unit.log instead of trusting a log_path + # directory. Kept because they cost nothing and do apply to any + # helper binary the suite runs. + # + # --save-log keeps /unit.log after each test. Without it + # test/conftest.py removes the per-restart temp dir, and with it the only + # copy of any report a forked daemon wrote -- which is what the gate + # below scans. pytest_opts is per-runtime (e.g. --restart for the + # graceful tests). - name: Run ${{ matrix.runtime }} sanitizer subset env: - ASAN_OPTIONS: "detect_leaks=0:log_path=${{ github.workspace }}/asan-logs/asan" - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:log_path=${{ github.workspace }}/asan-logs/ubsan" - run: | - python3 -m pytest -v ${{ matrix.pytest_opts }} ${{ matrix.tests }} - - - name: Show ASan/UBSan reports - if: always() + ASAN_OPTIONS: "detect_leaks=0" + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" run: | - if [ -n "$(ls -A asan-logs 2>/dev/null)" ]; then - for f in asan-logs/*; do - echo "===== ${f} =====" - cat "${f}" - done - else - echo "No ASan/UBSan report files were produced." - fi + python3 -m pytest -v --save-log ${{ matrix.pytest_opts }} ${{ matrix.tests }} - - name: Upload ASan/UBSan reports + - name: Upload unit logs if: always() uses: actions/upload-artifact@v4 with: - name: asan-ubsan-logs-${{ matrix.runtime }} - path: asan-logs/ - if-no-files-found: ignore - - # Sanitizer reports from forked children do not reliably fail pytest: the - # crash aborts a child while the session continues, and with log_path set - # the report never reaches the unit.log the harness scans. Fail the job - # explicitly on any report file, even when pytest was green. + name: unit-logs-${{ matrix.runtime }} + path: ${{ env.TMPDIR || '/tmp' }}/unit-test-*/unit.log + if-no-files-found: warn + + # Sanitizer reports do not reliably fail pytest. A UBSan violation that + # does not escalate to a signal fails nothing at all: the harness's + # Log.check_alerts() greps unit.log for "[alert]" and "Sanitizer", not + # for UBSan's "runtime error:". An ASan report does abort the router, + # but the session then fails with connection errors that read as an + # infrastructure flake rather than as a memory-safety bug. Measured on + # master: 24 runs of the configuration-churn workload produced a UBSan + # violation in every run with pytest green in 23 of them, and the one + # ASan heap-use-after-free surfaced only as "Client can't connect to the + # server". Fail the job explicitly on what the logs actually contain. - name: Fail on sanitizer reports if: always() run: | - if [ -n "$(ls -A asan-logs 2>/dev/null)" ]; then - echo "::error::AddressSanitizer/UBSan produced report(s) in the ${{ matrix.runtime }} leg; see the asan-ubsan-logs-${{ matrix.runtime }} artifact and the 'Show ASan/UBSan reports' step." - exit 1 - fi - echo "No sanitizer reports; ${{ matrix.runtime }} lifecycle paths are clean." + .github/scripts/check-sanitizer-reports.sh \ + "${TMPDIR:-/tmp}" .github/sanitizer-allowlist.txt diff --git a/test/conftest.py b/test/conftest.py index 008125653..77e60b050 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -476,6 +476,17 @@ def unit_run(state_dir=None): option.temp_dir = temporary_dir public_dir(temporary_dir) + # Every start gets a fresh temp dir and therefore a fresh, empty + # unit.log, so the read offsets recorded against the previous one are + # meaningless here. Teardown saves that offset unconditionally and only + # resets it when it removes the temp dir, which --save-log stops it from + # doing: without this, a --save-log --restart run seeks each test's reads + # to the size of the PREVIOUS test's log. Everything log-based then sees + # a tail slice or nothing at all -- Log.wait_for_record misses records + # that are present, and, worse silently, the teardown Log.check_alerts() + # and _check_fds() stop examining most of what they are given. + Log.pos.clear() + if oct(stat.S_IMODE(Path(builddir).stat().st_mode)) != '0o777': public_dir(builddir) From 15460929639513a88622f40b71b8fd1236480350 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 18 Aug 2026 16:33:23 +0200 Subject: [PATCH 010/391] test(process): require a replaced PID after the teardown churn The coverage claim of test_process_teardown_churn.py is that configuration rewrites really destroyed and respawned application processes while requests were in flight, and `assert len(pids) > 1` did not check it: the application is loaded with "spare": 2, so the very first `_app_pids()` sample the churn thread takes already holds two PIDs and the assertion passes on a run where nothing was ever replaced. The test would have degraded into a plain concurrency test without failing. Baseline the PIDs after the warm-up request and require a PID outside that set, which only a teardown and respawn can produce. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014og6SESdCDxXzCpR6H9X33 --- test/test_process_teardown_churn.py | 69 +++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/test/test_process_teardown_churn.py b/test/test_process_teardown_churn.py index ea5d72bba..fb4ffe8c7 100644 --- a/test/test_process_teardown_churn.py +++ b/test/test_process_teardown_churn.py @@ -52,6 +52,10 @@ # "settings" member (src/nxt_conf_validation.c: nxt_conf_vldt_setting_members). LISTEN_THREADS = 4 +# Prestarted workers. Named because the baseline below has to wait for +# exactly this many before it means anything. +SPARE = 2 + WORKERS = 4 REQUESTS_PER_WORKER = 16 CHURN_INTERVAL = 0.05 @@ -75,6 +79,25 @@ def _app_pids(): return {line.split()[0] for line in out.splitlines() if marker in line} +def _wait_for_pids(count, timeout=100): + """Poll until at least `count` application workers are visible. + + Returns the last sample either way, so a shortfall is reported by the + caller's assertion as the pid set it actually saw rather than as a bare + timeout. + """ + + for _ in range(timeout): + pids = _app_pids() + + if len(pids) >= count: + break + + time.sleep(0.1) + + return pids + + def test_process_teardown_churn(skip_fds_check): # Setting "listen_threads" changes the number of router worker engines, # and destroying an engine leaks its descriptors: measured on this tree, @@ -85,7 +108,7 @@ def test_process_teardown_churn(skip_fds_check): # causes, and it is unrelated to what the test is here to exercise. skip_fds_check(router=True) - client.load(APP, processes={"spare": 2, "max": 4, "idle_timeout": 5}) + client.load(APP, processes={"spare": SPARE, "max": 4, "idle_timeout": 5}) assert 'success' in client.conf( {"listen_threads": LISTEN_THREADS}, 'settings' @@ -93,6 +116,26 @@ def test_process_teardown_churn(skip_fds_check): assert client.post(body='ping', read_timeout=READ_TIMEOUT)['status'] == 200 + # The set of workers that exist *before* any configuration rewrite. The + # final assertion needs it to tell "the churn replaced processes" from + # "the churn changed nothing and these are the ones we started with". + # + # Wait for the configured spares rather than assuming the warm-up request + # brought them up. Unit starts spare processes asynchronously as the + # first one becomes idle, so an immediate sample can hold one of the two + # -- and the second would then appear in a later sample and be counted as + # a replacement, restoring the exact false positive the `fresh` check + # below exists to rule out. + baseline = _wait_for_pids(SPARE) + + # >=, not ==: "spare" is a floor, not a cap ("max" is 4), so the pool is + # allowed to be larger by the time the warm-up request has been served. + # What matters is that it is not still filling. + assert len(baseline) >= SPARE, ( + f'expected at least {SPARE} spare workers before the churn, ' + f'saw {sorted(baseline)}' + ) + lock = threading.Lock() stop = threading.Event() failures = [] @@ -225,9 +268,27 @@ def churn_worker(): assert resp['body'] == BODY, 'still serving intact body' # The workload has to have actually torn processes down, otherwise it is - # only a concurrency test. More than one PID means the application was - # destroyed and respawned while requests were in flight. - assert len(pids) > 1, f'application processes were not replaced: {pids}' + # only a concurrency test. A count is not enough to show that: "spare": 2 + # above means `baseline` already holds two PIDs, so `len(pids) > 1` passes + # on a run where no process was ever replaced. What proves a teardown and + # respawn is a PID that was not serving the application before the churn + # started. + fresh = pids - baseline + gone = baseline - _app_pids() + + assert fresh, ( + f'no new application process appeared: saw {sorted(pids)}, ' + f'baseline {sorted(baseline)}' + ) + + # A new pid on its own only proves the pool grew, which "max": 4 permits + # without any teardown at all. A pre-churn worker that is no longer + # serving is the direct evidence that the configuration rewrites actually + # tore an application down -- which is the whole point of the workload. + assert gone, ( + f'no pre-churn application process was torn down: ' + f'baseline {sorted(baseline)} all still serving' + ) # Requests must not have been shut out entirely, or nothing reached shared # memory concurrently with a teardown. From 13df5526fd564fd334d2c13b74cc4f755f952354 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 18 Aug 2026 16:33:41 +0200 Subject: [PATCH 011/391] test(process): make shared-memory bodies self-identifying Both teardown tests sent 'x' * 1 MiB and checked the echoed body against it. That check cannot see the single most likely corruption these tests exist to catch: a 16 KiB chunk (PORT_MMAP_CHUNK_SIZE, src/nxt_port_memory_int.h) recycled out from under one request and holding another *concurrent* request's payload. Both tests run four request threads on purpose, so that chunk is byte-identical to the bytes that belong there and the comparison passes. Only garbage that happens not to be 'x' was ever detected. Give every request a body built of 64-byte records stamped with the request's own tag and with the record's byte offset (test/unit/shm_body.py), so any splice is wrong within 64 bytes of where it starts, and the record header at the first wrong byte says whether the bytes came from another offset of this request (offset field differs) or from another request entirely (tag differs). The properties the tests rely on are kept: a short body is still a correct prefix and so still legitimate; the body is still 1 MiB, still spanning 64 chunks in each direction; and describe_mismatch() keeps the fail-open guard on the overlong-but-correct-prefix shape, which has no mismatching byte and would otherwise raise StopIteration out of the request loop. Bodies are built by one str.replace() over a skeleton that is generated once, so a 1 MiB body costs about 1.3 ms and the workload's runtime is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014og6SESdCDxXzCpR6H9X33 --- test/test_process_abrupt_teardown.py | 46 +++++----- test/test_process_teardown_churn.py | 46 +++++----- test/unit/shm_body.py | 124 +++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 42 deletions(-) create mode 100644 test/unit/shm_body.py diff --git a/test/test_process_abrupt_teardown.py b/test/test_process_abrupt_teardown.py index e31d154d3..3088148a3 100644 --- a/test/test_process_abrupt_teardown.py +++ b/test/test_process_abrupt_teardown.py @@ -45,6 +45,7 @@ from unit.applications.lang.php import ApplicationPHP from unit.log import Log +from unit.shm_body import describe_mismatch, make_body prerequisites = {'modules': {'php': 'all'}} @@ -57,8 +58,14 @@ # src/nxt_port_memory_int.h), so a request is inside the chunk allocator, and # the process/port lookups that reach it, for long enough that a kill on # another thread lands in the middle of one. +# +# The bytes are not uniform: every request carries a fresh body stamped, every +# 64 bytes, with that request's tag and the byte offset of the record (see +# unit/shm_body.py). WORKERS requests are in shared memory at once here, so a +# chunk recycled out from under one of them most likely holds another one's +# payload; against 'x' * BODY_SIZE that chunk is byte-identical to what +# belongs there and the check below cannot see it. BODY_SIZE = 1024 * 1024 -BODY = 'x' * BODY_SIZE # > 1 so more than one router worker engine is live; this is a top-level # "settings" member (src/nxt_conf_validation.c: nxt_conf_vldt_setting_members). @@ -163,11 +170,12 @@ def record(bucket, item): def request_worker(): while not stop.is_set(): + expected = make_body(BODY_SIZE) started = time.time() try: resp = client.post( - body=BODY, + body=expected, read_timeout=READ_TIMEOUT, read_buffer_size=65536, ) @@ -191,26 +199,20 @@ def request_worker(): # A short body is legitimate: the worker can die after its # headers are on the wire. Wrong *bytes* are not - that is # what reading freed or recycled shared memory looks like. - if body != BODY[: len(body)]: - # next() needs a default. A body that is all 'x' but - # LONGER than the request still fails the comparison - # above -- BODY[:len(body)] clamps to BODY -- and then - # has no mismatching byte to find. Without the default - # that raises StopIteration here, outside the try that - # guards the request, killing the worker thread instead - # of recording the corruption: the detector fails open - # on one of the shapes it exists to catch. - at = next( - (i for i, c in enumerate(body) if c != 'x'), None - ) + if body != expected[: len(body)]: + # describe_mismatch() reports the first wrong byte and the + # record stamp found there, so a failure says whether the + # bytes came from another offset of this request or from + # another request entirely. It keeps the fail-open guard + # too: the overlong-but-correct-prefix shape has no + # mismatching byte, and a bare next() would raise + # StopIteration here, outside the try that guards the + # request, killing this thread instead of recording the + # corruption. record( failures, f'corrupt body: {len(body)} bytes, ' - + ( - f'first mismatch at {at}' - if at is not None - else f'uniform but longer than {BODY_SIZE}' - ), + + describe_mismatch(body, expected), ) continue @@ -267,13 +269,15 @@ def request_worker(): assert not failures, f'{len(failures)} failure(s), first: {failures[0]}' # The daemon is still alive and still serves the whole payload correctly. + expected = make_body(BODY_SIZE) + resp = client.post( - body=BODY, read_timeout=READ_TIMEOUT, read_buffer_size=65536 + body=expected, read_timeout=READ_TIMEOUT, read_buffer_size=65536 ) assert resp['status'] == 200, 'still serving' assert len(resp['body']) == BODY_SIZE, 'still serving whole body' - assert resp['body'] == BODY, 'still serving intact body' + assert resp['body'] == expected, 'still serving intact body' # Requests must not have been shut out entirely, or nothing was in flight # when a worker died. diff --git a/test/test_process_teardown_churn.py b/test/test_process_teardown_churn.py index fb4ffe8c7..2af4fd65e 100644 --- a/test/test_process_teardown_churn.py +++ b/test/test_process_teardown_churn.py @@ -32,6 +32,7 @@ from unit.applications.lang.php import ApplicationPHP from unit.log import Log +from unit.shm_body import describe_mismatch, make_body prerequisites = {'modules': {'php': 'all'}} @@ -45,8 +46,14 @@ # chunks in each direction and keeps the worker inside the chunk allocator # (nxt_port_mmap_get_buf()/nxt_port_mmap_increase_buf()) long enough for a # teardown on another thread to overlap it. +# +# The bytes are not uniform: every request carries a fresh body stamped, every +# 64 bytes, with that request's tag and the byte offset of the record (see +# unit/shm_body.py). WORKERS requests are in shared memory at once here, so a +# chunk recycled out from under one of them most likely holds another one's +# payload; against 'x' * BODY_SIZE that chunk is byte-identical to what +# belongs there and the check below cannot see it. BODY_SIZE = 1024 * 1024 -BODY = 'x' * BODY_SIZE # > 1 so more than one router worker engine is live; this is a top-level # "settings" member (src/nxt_conf_validation.c: nxt_conf_vldt_setting_members). @@ -148,11 +155,12 @@ def record(bucket, item): def request_worker(): for _ in range(REQUESTS_PER_WORKER): + expected = make_body(BODY_SIZE) started = time.time() try: resp = client.post( - body=BODY, + body=expected, read_timeout=READ_TIMEOUT, read_buffer_size=65536, ) @@ -173,26 +181,20 @@ def request_worker(): # A short body is legitimate: the application can die after its # headers are on the wire. Wrong *bytes* are not - that is # what reading freed or recycled shared memory looks like. - if body != BODY[: len(body)]: - # next() needs a default. A body that is all 'x' but - # LONGER than the request still fails the comparison - # above -- BODY[:len(body)] clamps to BODY -- and then - # has no mismatching byte to find. Without the default - # that raises StopIteration here, outside the try that - # guards the request, killing the worker thread instead - # of recording the corruption: the detector fails open - # on one of the shapes it exists to catch. - at = next( - (i for i, c in enumerate(body) if c != 'x'), None - ) + if body != expected[: len(body)]: + # describe_mismatch() reports the first wrong byte and the + # record stamp found there, so a failure says whether the + # bytes came from another offset of this request or from + # another request entirely. It keeps the fail-open guard + # too: the overlong-but-correct-prefix shape has no + # mismatching byte, and a bare next() would raise + # StopIteration here, outside the try that guards the + # request, killing this thread instead of recording the + # corruption. record( failures, f'corrupt body: {len(body)} bytes, ' - + ( - f'first mismatch at {at}' - if at is not None - else f'uniform but longer than {BODY_SIZE}' - ), + + describe_mismatch(body, expected), ) continue @@ -260,12 +262,14 @@ def churn_worker(): assert not failures, f'{len(failures)} failure(s), first: {failures[0]}' # The daemon is still alive and still serves the whole payload correctly. - resp = client.post(body=BODY, read_timeout=READ_TIMEOUT, + expected = make_body(BODY_SIZE) + + resp = client.post(body=expected, read_timeout=READ_TIMEOUT, read_buffer_size=65536) assert resp['status'] == 200, 'still serving' assert len(resp['body']) == BODY_SIZE, 'still serving whole body' - assert resp['body'] == BODY, 'still serving intact body' + assert resp['body'] == expected, 'still serving intact body' # The workload has to have actually torn processes down, otherwise it is # only a concurrency test. A count is not enough to show that: "spare": 2 diff --git a/test/unit/shm_body.py b/test/unit/shm_body.py new file mode 100644 index 000000000..bd71c6207 --- /dev/null +++ b/test/unit/shm_body.py @@ -0,0 +1,124 @@ +"""Self-identifying request bodies for the shared-memory teardown tests. + +Why not ``'x' * SIZE``: test_process_teardown_churn.py and +test_process_abrupt_teardown.py exist to notice the router serving freed or +recycled shared memory, and the most likely shape of that is a 16 KiB chunk +(PORT_MMAP_CHUNK_SIZE, src/nxt_port_memory_int.h) belonging to a *different* +concurrent request turning up in this one. Against a uniform body such a +chunk is byte-identical to the bytes that belong there, so the integrity +check cannot see it at all: it only ever catches garbage that happens not to +be 'x'. Both tests run several request threads on purpose, so a cross-request +chunk is exactly the corruption they are best placed to catch and exactly the +one a uniform body hides. + +The body is a run of fixed-width records, each stamped with the request's own +tag and with its own byte offset: + + | tag: 8 hex digits | offset: 8 hex digits | 48 filler letters | + +so every 64 bytes of the payload say both *which* request the bytes came from +and *where in it* they belong. A chunk swapped in from another request +mismatches on the tag; a chunk from another offset of the same request +mismatches on the offset field; a shift of less than one record mismatches on +the filler letter, which advances with the record index. Byte-wise, that +means any splice at a chunk boundary is wrong within at most 64 bytes of the +splice, and the record header at the first wrong byte names the culprit. + +Cost matters - these tests send 64 to 100 bodies of 1 MiB each and are meant +to stay at a few seconds - so the offset skeleton is built once per size at +import time and a per-request body is a single str.replace() of the tag +placeholder: one C-speed pass over 1 MiB instead of a Python loop. +""" + +import itertools + +# One record per 64 bytes: short enough that a corrupt region is localised to +# within 64 bytes of where it starts, long enough that a 1 MiB body is 16384 +# records rather than ten times that. +RECORD_SIZE = 64 + +TAG_LEN = 8 +OFFSET_LEN = 8 + +_FILLER_LEN = RECORD_SIZE - TAG_LEN - OFFSET_LEN + +_LETTERS = 'abcdefghijklmnopqrstuvwxyz' + +# Tags are hex digits, so '~' cannot occur in a finished body and replacing +# the placeholder cannot corrupt payload that happens to look like it. +_PLACEHOLDER = '~' * TAG_LEN + +_counter = itertools.count(1) + +_skeletons = {} + + +def _skeleton(size): + if size % RECORD_SIZE: + raise ValueError(f'size must be a multiple of {RECORD_SIZE}') + + if size not in _skeletons: + _skeletons[size] = ''.join( + _PLACEHOLDER + + f'{offset:0{OFFSET_LEN}X}' + + _LETTERS[(offset // RECORD_SIZE) % len(_LETTERS)] * _FILLER_LEN + for offset in range(0, size, RECORD_SIZE) + ) + + return _skeletons[size] + + +def make_body(size): + """A body of `size` bytes, distinguishable from every other body this + process has handed out. + + next() on an itertools.count is a single bytecode under the GIL, so + request threads can call this without a lock, and consecutive tags always + differ in at least one hex digit - which is all the byte-wise comparison + in the tests needs to tell two in-flight requests apart. + """ + + return _skeleton(size).replace(_PLACEHOLDER, f'{next(_counter):08X}') + + +def describe_mismatch(body, expected): + """One line saying how `body` differs from `expected`, for a failure + message. + + Reports the first wrong byte and the record header found there against + the header that belongs there, which is what separates "a chunk from + another request" (tag differs) from "a chunk from another offset of this + request" (offset differs) from plain garbage (neither parses). + """ + + # next() needs a default. zip() stops at the shorter of the two, so a + # body that is a correct prefix but LONGER than the request - which still + # fails the caller's `body != expected[:len(body)]` test, since that + # slice clamps to `expected` - has no mismatching byte to find here. A + # bare next() would raise StopIteration out of the caller's request loop + # and kill the thread instead of recording the corruption: the detector + # would fail open on one of the shapes it exists to catch. + at = next( + (i for i, (c, e) in enumerate(zip(body, expected)) if c != e), None + ) + + if at is None: + return f'correct prefix but longer than {len(expected)} bytes' + + return ( + f'first mismatch at {at} ({body[at]!r} != {expected[at]!r}): ' + f'got {_record_header(body, at)}, ' + f'want {_record_header(expected, at)}' + ) + + +def _record_header(text, at): + """The tag/offset stamp of the record containing byte `at`.""" + + start = at - at % RECORD_SIZE + head = text[start : start + TAG_LEN + OFFSET_LEN] + + if len(head) < TAG_LEN + OFFSET_LEN: + return f'' + + return f'tag={head[:TAG_LEN]} offset={head[TAG_LEN:]}' From 02cd8d084da1aed3e30a383e58b4d7e6504ec8a5 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 18 Aug 2026 16:55:28 +0200 Subject: [PATCH 012/391] config: JSON Pointer paths and "did you mean" in validation errors Configuration validation errors say what is wrong but not where. For a mistyped or misplaced member in a deeply nested object the caller gets a message and a byte offset, and has to count braces to find the member. Carry an RFC 6901 JSON Pointer to the offending location in location.path, and when a rejected member name is a close typo of a known schema key, a "did you mean" name in suggestion. Strictly additive: existing response fields and error message wording are unchanged, and both new fields are omitted when they do not apply. * src/nxt_conf.h: extend nxt_conf_validation_t with a path linked list built from stack nodes, plus the two rendered output fields. Not exported. * src/nxt_conf_validation.c: - nxt_conf_vldt_render_pointer() walks the path list and escapes '~' -> "~0" and '/' -> "~1" per RFC 6901. "" is the document root. It is folded into nxt_conf_vldt_error(), so every existing error call-site gets a pointer without touching its message text -- and error sites added later get one for free. - Path segments are pushed and popped at the three recursion sites: nxt_conf_vldt_object(), nxt_conf_vldt_object_iterator() and nxt_conf_vldt_array_iterator() (decimal index), and around the NXT_CONF_VLDT_REQUIRED and NXT_CONF_VLDT_TSTR errors so those attribute to the member rather than to its parent container. - nxt_conf_vldt_edit_distance() is Damerau-Levenshtein over bounded stack buffers (NXT_CONF_VLDT_EDIT_MAX = 64); longer names simply get no suggestion. No heap, no allocation on the compare path. - Rendering never fails validation: on an allocation failure or a size bound violation pointer.start is left NULL and the response carries no path, rather than a false root pointer. * src/nxt_controller.c: emit location.path and the top-level suggestion. The location object is now built by counting the members it will actually hold, since a pointer can be present without an offset and vice versa. The pointer is allocated from vldt->pool, which is the connection pool at both request-path call sites, so it outlives the nxt_mp_destroy() of the conf pool that backs the parsed document. Co-authored-by: Gemini 3.1 Pro Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ADLFQU9cw2E1EKeJgLDB21 --- src/nxt_conf.h | 11 + src/nxt_conf_validation.c | 483 +++++++++++++++++++++++--- src/nxt_controller.c | 57 ++- test/test_config_validation_errors.py | 212 +++++++++++ test/test_tls_conf_command.py | 23 ++ 5 files changed, 730 insertions(+), 56 deletions(-) create mode 100644 test/test_config_validation_errors.py diff --git a/src/nxt_conf.h b/src/nxt_conf.h index 493d64801..60a2061bc 100644 --- a/src/nxt_conf.h +++ b/src/nxt_conf.h @@ -68,6 +68,14 @@ typedef struct { } nxt_conf_json_pretty_t; +typedef struct nxt_conf_vldt_path_s nxt_conf_vldt_path_t; + +struct nxt_conf_vldt_path_s { + nxt_conf_vldt_path_t *prev; + nxt_str_t seg; +}; + + typedef struct { nxt_conf_value_t *conf; nxt_mp_t *pool; @@ -76,6 +84,9 @@ typedef struct { nxt_tstr_state_t *tstr_state; nxt_mp_t *conf_pool; nxt_uint_t ver; + nxt_conf_vldt_path_t *path; /* Stack (top = deepest segment). */ + nxt_str_t pointer; /* RFC 6901 pointer at error site. */ + nxt_str_t suggestion; /* "Did you mean X" member name. */ } nxt_conf_validation_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 8334ee10e..7bf263632 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -75,8 +75,12 @@ struct nxt_conf_vldt_object_s { static nxt_int_t nxt_conf_vldt_type(nxt_conf_validation_t *vldt, const nxt_str_t *name, nxt_conf_value_t *value, nxt_conf_vldt_type_t type); +static nxt_int_t nxt_conf_vldt_verror(nxt_conf_validation_t *vldt, + const char *fmt, va_list args); static nxt_int_t nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...); +static nxt_int_t nxt_conf_vldt_member_error(nxt_conf_validation_t *vldt, + const nxt_str_t *member, const char *fmt, ...); static nxt_int_t nxt_conf_vldt_var(nxt_conf_validation_t *vldt, const nxt_str_t *name, nxt_str_t *value); static nxt_int_t nxt_conf_vldt_if(nxt_conf_validation_t *vldt, @@ -1615,6 +1619,14 @@ nxt_conf_validate(nxt_conf_validation_t *vldt) ret = nxt_tstr_state_done(vldt->tstr_state, error); if (ret != NXT_OK) { ret = nxt_conf_vldt_error(vldt, "%s", error); + + /* + * Deferred tstr compilation runs once the traversal has unwound, so + * vldt->path no longer names the value that failed. An empty pointer + * here would claim the document root; report no location instead. + */ + vldt->pointer.start = NULL; + return ret; } @@ -1774,17 +1786,91 @@ nxt_conf_vldt_type(nxt_conf_validation_t *vldt, const nxt_str_t *name, } +/* + * Serialize vldt->path into vldt->pointer as an RFC 6901 JSON Pointer. + * Root is the empty string "". Segments are separated by '/'. Within each + * segment, '~' is encoded as "~0" and '/' as "~1" (in that order). + * + * Never fails validation: on allocation failure the pointer is left empty. + */ +static void +nxt_conf_vldt_render_pointer(nxt_conf_validation_t *vldt) +{ + u_char *p; + size_t size, n; + u_char c; + nxt_conf_vldt_path_t *node; + + static u_char empty[1] = ""; + + /* + * Always produce a non-NULL pointer: the empty string "" is the + * RFC 6901 pointer to the document root. Callers distinguish + * "validation error, pointer available" from "no pointer" via + * start != NULL. + */ + vldt->pointer.length = 0; + vldt->pointer.start = empty; + + if (vldt->path == NULL) { + return; + } + + size = 0; + for (node = vldt->path; node != NULL; node = node->prev) { + if (nxt_slow_path(size == NXT_SIZE_T_MAX)) { + vldt->pointer.start = NULL; + return; + } + size += 1; /* leading '/' */ + for (n = 0; n < node->seg.length; n++) { + c = node->seg.start[n]; + if (nxt_slow_path(size >= NXT_SIZE_T_MAX - 1)) { + vldt->pointer.start = NULL; + return; + } + size += (c == '~' || c == '/') ? 2 : 1; + } + } + + p = nxt_mp_nget(vldt->pool, size); + if (p == NULL) { + vldt->pointer.start = NULL; + return; + } + + vldt->pointer.start = p; + vldt->pointer.length = size; + + p += size; + + for (node = vldt->path; node != NULL; node = node->prev) { + for (n = node->seg.length; n > 0; n--) { + c = node->seg.start[n - 1]; + if (c == '~') { + *--p = '0'; + *--p = '~'; + } else if (c == '/') { + *--p = '1'; + *--p = '~'; + } else { + *--p = c; + } + } + *--p = '/'; + } +} + + static nxt_int_t -nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...) +nxt_conf_vldt_verror(nxt_conf_validation_t *vldt, const char *fmt, + va_list args) { u_char *p, *end; size_t size; - va_list args; u_char error[NXT_MAX_ERROR_STR]; - va_start(args, fmt); end = nxt_vsprintf(error, error + NXT_MAX_ERROR_STR, fmt, args); - va_end(args); size = end - error; @@ -1798,10 +1884,187 @@ nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...) vldt->error.length = size; vldt->error.start = p; + nxt_conf_vldt_render_pointer(vldt); + return NXT_DECLINED; } +static nxt_int_t +nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...) +{ + va_list args; + nxt_int_t ret; + + va_start(args, fmt); + ret = nxt_conf_vldt_verror(vldt, fmt, args); + va_end(args); + + return ret; +} + + +/* + * Report an error against a member of the object being validated, for the + * semantic checks that run once nxt_conf_vldt_object() has finished the + * traversal and already popped that member's path segment. Without this the + * location would name the containing object instead of the value that failed. + */ + +static nxt_int_t +nxt_conf_vldt_member_error(nxt_conf_validation_t *vldt, + const nxt_str_t *member, const char *fmt, ...) +{ + va_list args; + nxt_int_t ret; + nxt_conf_vldt_path_t seg; + + seg.prev = vldt->path; + seg.seg = *member; + vldt->path = &seg; + + va_start(args, fmt); + ret = nxt_conf_vldt_verror(vldt, fmt, args); + va_end(args); + + vldt->path = seg.prev; + + return ret; +} + + +/* + * Damerau-Levenshtein distance between two byte strings. Rejects inputs + * that would overflow the stack buffer by returning SIZE_MAX. + * + * Used only inside validation error paths; input lengths are bounded by + * configuration member-name lengths. + */ +#define NXT_CONF_VLDT_EDIT_MAX 64 + +static size_t +nxt_conf_vldt_edit_distance(const nxt_str_t *a, const nxt_str_t *b) +{ + size_t i, j, cost, ins, del, sub, tr, min; + size_t prev2[NXT_CONF_VLDT_EDIT_MAX + 1]; + size_t prev[NXT_CONF_VLDT_EDIT_MAX + 1]; + size_t curr[NXT_CONF_VLDT_EDIT_MAX + 1]; + + if (a->length > NXT_CONF_VLDT_EDIT_MAX + || b->length > NXT_CONF_VLDT_EDIT_MAX) + { + return (size_t) -1; + } + + for (j = 0; j <= b->length; j++) { + prev[j] = j; + prev2[j] = 0; + } + + for (i = 1; i <= a->length; i++) { + curr[0] = i; + + for (j = 1; j <= b->length; j++) { + cost = (a->start[i - 1] == b->start[j - 1]) ? 0 : 1; + + ins = curr[j - 1] + 1; /* left neighbour */ + del = prev[j] + 1; /* upper neighbour */ + sub = prev[j - 1] + cost; /* diagonal */ + + min = ins < del ? ins : del; + if (sub < min) { + min = sub; + } + + if (i > 1 && j > 1 + && a->start[i - 1] == b->start[j - 2] + && a->start[i - 2] == b->start[j - 1]) + { + tr = prev2[j - 2] + 1; + if (tr < min) { + min = tr; + } + } + + curr[j] = min; + } + + for (j = 0; j <= b->length; j++) { + prev2[j] = prev[j]; + prev[j] = curr[j]; + } + } + + return prev[b->length]; +} + + +/* + * Emit "Unknown parameter" error for a member not present in the schema + * table. If a sufficiently-close, uniquely-best candidate exists, store + * its name in vldt->suggestion so the response body can surface it. + * + * Does NOT modify the legacy error-message wording. + */ +static nxt_int_t +nxt_conf_vldt_unknown_member(nxt_conf_validation_t *vldt, + const nxt_str_t *name, nxt_conf_vldt_object_t *vals) +{ + size_t d, best, second; + size_t threshold, shorter; + nxt_conf_vldt_object_t *v, *best_v; + + best = (size_t) -1; + second = (size_t) -1; + best_v = NULL; + + v = vals; + + for ( ;; ) { + if (v->name.length == 0) { + if (v->u.members != NULL) { + v = v->u.members; + continue; + } + break; + } + + d = nxt_conf_vldt_edit_distance(name, &v->name); + if (d < best) { + second = best; + best = d; + best_v = v; + } else if (d < second) { + second = d; + } + + v++; + } + + threshold = nxt_max((size_t) 2, name->length / 3); + + /* + * Require the edit to cover at most half of the shorter name as well as + * to stay under the threshold. The threshold alone is too permissive for + * short names: at two bytes it admits a distance of 2, which is every + * two-letter member in the schema, so "zz" would "did you mean" its way to + * an unrelated name it shares no characters with. + * + * best < second keeps the match unambiguous. + */ + + if (best_v != NULL && best <= threshold && best < second) { + shorter = nxt_min(name->length, best_v->name.length); + + if (best * 2 <= shorter) { + vldt->suggestion = best_v->name; + } + } + + return nxt_conf_vldt_error(vldt, "Unknown parameter \"%V\".", name); +} + + nxt_inline nxt_int_t nxt_conf_vldt_unsupported(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) @@ -2883,10 +3146,11 @@ static nxt_int_t nxt_conf_vldt_object_conf_commands(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) { - uint32_t index; - nxt_int_t ret; - nxt_str_t name; - nxt_conf_value_t *member; + uint32_t index; + nxt_int_t ret; + nxt_str_t name; + nxt_conf_value_t *member; + nxt_conf_vldt_path_t seg; index = 0; @@ -2897,7 +3161,21 @@ nxt_conf_vldt_object_conf_commands(nxt_conf_validation_t *vldt, break; } + /* + * This iterates the object itself rather than going through + * nxt_conf_vldt_object_iterator(), so the command name has to be + * pushed here; without it the error below would point at + * conf_commands rather than at the command that failed. + */ + + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + ret = nxt_conf_vldt_type(vldt, &name, member, NXT_CONF_VLDT_STRING); + + vldt->path = seg.prev; + if (ret != NXT_OK) { return ret; } @@ -3097,8 +3375,8 @@ nxt_conf_vldt_listen_backlog(nxt_conf_validation_t *vldt, static nxt_int_t -nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, - nxt_conf_value_t *value) +nxt_conf_vldt_app_type(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + nxt_app_lang_module_t **langp) { nxt_int_t ret; nxt_str_t type; @@ -3108,6 +3386,47 @@ nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, static const nxt_str_t type_str = nxt_string("type"); + type_value = nxt_conf_get_object_member(value, &type_str, NULL); + + if (type_value == NULL) { + return nxt_conf_vldt_error(vldt, + "Application must have the \"type\" property set."); + } + + ret = nxt_conf_vldt_type(vldt, &type_str, type_value, NXT_CONF_VLDT_STRING); + + if (ret != NXT_OK) { + return ret; + } + + nxt_conf_get_string(type_value, &type); + + thread = nxt_thread(); + + lang = nxt_app_lang_module(thread->runtime, &type); + if (lang == NULL) { + return nxt_conf_vldt_error(vldt, + "The module to run \"%V\" is not found " + "among the available application modules.", + &type); + } + + *langp = lang; + + return NXT_OK; +} + + +static nxt_int_t +nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, + nxt_conf_value_t *value) +{ + nxt_int_t ret; + nxt_conf_vldt_path_t seg; + nxt_app_lang_module_t *lang; + + static const nxt_str_t type_str = nxt_string("type"); + static const struct { nxt_conf_vldt_handler_t validator; nxt_conf_vldt_object_t *members; @@ -3123,37 +3442,33 @@ nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, { nxt_conf_vldt_object, nxt_conf_vldt_wasm_wc_members }, }; + lang = NULL; + ret = nxt_conf_vldt_type(vldt, name, value, NXT_CONF_VLDT_OBJECT); if (ret != NXT_OK) { return ret; } - type_value = nxt_conf_get_object_member(value, &type_str, NULL); + /* + * The three checks below all concern the "type" member, so they report + * against it rather than against the application object that contains + * it. The nested validator that follows must run with the path back at + * the application, so the segment is popped before it. + */ - if (type_value == NULL) { - return nxt_conf_vldt_error(vldt, - "Application must have the \"type\" property set."); - } + seg.prev = vldt->path; + seg.seg = type_str; + vldt->path = &seg; - ret = nxt_conf_vldt_type(vldt, &type_str, type_value, NXT_CONF_VLDT_STRING); + ret = nxt_conf_vldt_app_type(vldt, value, &lang); + + vldt->path = seg.prev; if (ret != NXT_OK) { return ret; } - nxt_conf_get_string(type_value, &type); - - thread = nxt_thread(); - - lang = nxt_app_lang_module(thread->runtime, &type); - if (lang == NULL) { - return nxt_conf_vldt_error(vldt, - "The module to run \"%V\" is not found " - "among the available application modules.", - &type); - } - return types[lang->type].validator(vldt, value, types[lang->type].members); } @@ -3167,6 +3482,7 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_str_t name, var; nxt_conf_value_t *member; nxt_conf_vldt_object_t *vals; + nxt_conf_vldt_path_t seg; vals = data; @@ -3185,8 +3501,16 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, member = nxt_conf_get_object_member(value, &vals->name, NULL); if (member == NULL) { - return nxt_conf_vldt_error(vldt, "Required parameter \"%V\" " - "is missing.", &vals->name); + seg.prev = vldt->path; + seg.seg = vals->name; + vldt->path = &seg; + + ret = nxt_conf_vldt_error(vldt, "Required parameter \"%V\" " + "is missing.", &vals->name); + + vldt->path = seg.prev; + + return ret; } } @@ -3212,8 +3536,7 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, continue; } - return nxt_conf_vldt_error(vldt, "Unknown parameter \"%V\".", - &name); + return nxt_conf_vldt_unknown_member(vldt, &name, data); } if (!nxt_strstr_eq(&vals->name, &name)) { @@ -3227,7 +3550,14 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_conf_get_string(member, &var); if (nxt_is_tstr(&var)) { + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + ret = nxt_conf_vldt_var(vldt, &name, &var); + + vldt->path = seg.prev; + if (ret != NXT_OK) { return ret; } @@ -3236,8 +3566,13 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } } + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + ret = nxt_conf_vldt_type(vldt, &name, member, vals->type); if (ret != NXT_OK) { + vldt->path = seg.prev; return ret; } @@ -3245,10 +3580,12 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, ret = vals->validator(vldt, member, vals->u.members); if (ret != NXT_OK) { + vldt->path = seg.prev; return ret; } } + vldt->path = seg.prev; break; } } @@ -3291,6 +3628,10 @@ nxt_conf_vldt_processes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_int_t ret; nxt_conf_vldt_processes_conf_t proc; + static const nxt_str_t spare_str = nxt_string("spare"); + static const nxt_str_t max_str = nxt_string("max"); + static const nxt_str_t idle_timeout_str = nxt_string("idle_timeout"); + if (nxt_conf_type(value) == NXT_CONF_INTEGER) { int_value = nxt_conf_get_number(value); @@ -3325,37 +3666,44 @@ nxt_conf_vldt_processes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } if (proc.spare < 0) { - return nxt_conf_vldt_error(vldt, "The \"spare\" number must not be " + return nxt_conf_vldt_member_error(vldt, &spare_str, + "The \"spare\" number must not be " "negative."); } if (proc.spare > NXT_INT32_T_MAX) { - return nxt_conf_vldt_error(vldt, "The \"spare\" number must not " + return nxt_conf_vldt_member_error(vldt, &spare_str, + "The \"spare\" number must not " "exceed %d.", NXT_INT32_T_MAX); } if (proc.max < 1) { - return nxt_conf_vldt_error(vldt, "The \"max\" number must be equal " + return nxt_conf_vldt_member_error(vldt, &max_str, + "The \"max\" number must be equal " "to or greater than 1."); } if (proc.max > NXT_INT32_T_MAX) { - return nxt_conf_vldt_error(vldt, "The \"max\" number must not " + return nxt_conf_vldt_member_error(vldt, &max_str, + "The \"max\" number must not " "exceed %d.", NXT_INT32_T_MAX); } if (proc.max < proc.spare) { - return nxt_conf_vldt_error(vldt, "The \"spare\" number must be " + return nxt_conf_vldt_member_error(vldt, &spare_str, + "The \"spare\" number must be " "less than or equal to \"max\"."); } if (proc.idle_timeout < 0) { - return nxt_conf_vldt_error(vldt, "The \"idle_timeout\" number must not " + return nxt_conf_vldt_member_error(vldt, &idle_timeout_str, + "The \"idle_timeout\" number must not " "be negative."); } if (proc.idle_timeout > NXT_INT32_T_MAX / 1000) { - return nxt_conf_vldt_error(vldt, "The \"idle_timeout\" number must not " + return nxt_conf_vldt_member_error(vldt, &idle_timeout_str, + "The \"idle_timeout\" number must not " "exceed %d.", NXT_INT32_T_MAX / 1000); } @@ -3371,6 +3719,7 @@ nxt_conf_vldt_object_iterator(nxt_conf_validation_t *vldt, nxt_int_t ret; nxt_str_t name; nxt_conf_value_t *member; + nxt_conf_vldt_path_t seg; nxt_conf_vldt_member_t validator; validator = (nxt_conf_vldt_member_t) data; @@ -3382,7 +3731,14 @@ nxt_conf_vldt_object_iterator(nxt_conf_validation_t *vldt, return NXT_OK; } + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + ret = validator(vldt, &name, member); + + vldt->path = seg.prev; + if (ret != NXT_OK) { return ret; } @@ -3394,10 +3750,13 @@ static nxt_int_t nxt_conf_vldt_array_iterator(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) { + u_char *p; uint32_t index; nxt_int_t ret; nxt_conf_value_t *element; + nxt_conf_vldt_path_t seg; nxt_conf_vldt_element_t validator; + u_char buf[NXT_INT_T_LEN]; validator = (nxt_conf_vldt_element_t) data; @@ -3408,8 +3767,17 @@ nxt_conf_vldt_array_iterator(nxt_conf_validation_t *vldt, return NXT_OK; } + p = nxt_sprintf(buf, buf + sizeof(buf), "%uD", index); + + seg.prev = vldt->path; + seg.seg.start = buf; + seg.seg.length = p - buf; + vldt->path = &seg; + ret = validator(vldt, element); + vldt->path = seg.prev; + if (ret != NXT_OK) { return ret; } @@ -3832,8 +4200,9 @@ static nxt_int_t nxt_conf_vldt_upstream(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value) { - nxt_int_t ret; - nxt_conf_value_t *conf; + nxt_int_t ret; + nxt_conf_value_t *conf; + nxt_conf_vldt_path_t seg; static const nxt_str_t servers = nxt_string("servers"); @@ -3851,8 +4220,19 @@ nxt_conf_vldt_upstream(nxt_conf_validation_t *vldt, nxt_str_t *name, conf = nxt_conf_get_object_member(value, &servers, NULL); if (conf == NULL) { - return nxt_conf_vldt_error(vldt, "The \"%V\" upstream must contain " - "\"servers\" object value.", name); + /* Same convention as the NXT_CONF_VLDT_REQUIRED check: a missing + * member is reported against the member, not its container. */ + + seg.prev = vldt->path; + seg.seg = servers; + vldt->path = &seg; + + ret = nxt_conf_vldt_error(vldt, "The \"%V\" upstream must contain " + "\"servers\" object value.", name); + + vldt->path = seg.prev; + + return ret; } return NXT_OK; @@ -3989,9 +4369,11 @@ nxt_conf_vldt_access_log(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) { nxt_int_t ret; + nxt_conf_vldt_path_t seg; nxt_conf_vldt_access_log_conf_t conf; static const nxt_str_t format_str = nxt_string("format"); + static const nxt_str_t path_str = nxt_string("path"); if (nxt_conf_type(value) == NXT_CONF_STRING) { return nxt_conf_vldt_c_string(vldt, value, (void *) "access_log"); @@ -4013,19 +4395,28 @@ nxt_conf_vldt_access_log(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } if (conf.path.length == 0) { - return nxt_conf_vldt_error(vldt, + return nxt_conf_vldt_member_error(vldt, &path_str, "The \"path\" string must not be empty."); } /* The log path is opened as a NUL-terminated C string. */ if (memchr(conf.path.start, '\0', conf.path.length) != NULL) { - return nxt_conf_vldt_error(vldt, "The \"path\" value must not " + return nxt_conf_vldt_member_error(vldt, &path_str, + "The \"path\" value must not " "contain null character."); } if (nxt_is_tstr(&conf.format)) { - return nxt_conf_vldt_var(vldt, &format_str, &conf.format); + seg.prev = vldt->path; + seg.seg = format_str; + vldt->path = &seg; + + ret = nxt_conf_vldt_var(vldt, &format_str, &conf.format); + + vldt->path = seg.prev; + + return ret; } return NXT_OK; diff --git a/src/nxt_controller.c b/src/nxt_controller.c index c8f418f47..b46d19570 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -38,6 +38,8 @@ typedef struct { ssize_t offset; nxt_uint_t line; nxt_uint_t column; + nxt_str_t pointer; /* RFC 6901 path to validation error. */ + nxt_str_t suggestion; /* "Did you mean" member name. */ } nxt_controller_response_t; @@ -1504,6 +1506,8 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, if (rc == NXT_DECLINED) { resp.detail = vldt.error; + resp.pointer = vldt.pointer; + resp.suggestion = vldt.suggestion; goto invalid_conf; } @@ -1587,6 +1591,8 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, if (rc == NXT_DECLINED) { resp.detail = vldt.error; + resp.pointer = vldt.pointer; + resp.suggestion = vldt.suggestion; goto invalid_conf; } @@ -2630,6 +2636,8 @@ nxt_controller_response(nxt_task_t *task, nxt_controller_request_t *req, static const nxt_str_t offset_str = nxt_string("offset"); static const nxt_str_t line_str = nxt_string("line"); static const nxt_str_t column_str = nxt_string("column"); + static const nxt_str_t path_str = nxt_string("path"); + static const nxt_str_t suggestion_str = nxt_string("suggestion"); static nxt_time_string_t date_cache = { (nxt_atomic_uint_t) -1, @@ -2667,9 +2675,15 @@ nxt_controller_response(nxt_task_t *task, nxt_controller_request_t *req, value = resp->conf; if (value == NULL) { + nxt_uint_t loc_n, loc_i, have_offset, have_pointer; + + have_offset = (resp->status >= 400 && resp->offset != -1); + have_pointer = (resp->status >= 400 && resp->pointer.start != NULL); + n = 1 + (resp->detail.length != 0) - + (resp->status >= 400 && resp->offset != -1); + + (have_offset || have_pointer) + + (resp->suggestion.length != 0); value = nxt_conf_create_object(c->mem_pool, n); @@ -2696,24 +2710,47 @@ nxt_controller_response(nxt_task_t *task, nxt_controller_request_t *req, nxt_conf_set_member_string(value, &detail_str, &resp->detail, n); } - if (resp->status >= 400 && resp->offset != -1) { + if (have_offset || have_pointer) { n++; - location = nxt_conf_create_object(c->mem_pool, - resp->line != 0 ? 3 : 1); + loc_n = (have_offset ? (resp->line != 0 ? 3 : 1) : 0) + have_pointer; + + location = nxt_conf_create_object(c->mem_pool, loc_n); + + if (nxt_slow_path(location == NULL)) { + nxt_controller_conn_close(task, c, req); + return; + } nxt_conf_set_member(value, &location_str, location, n); - nxt_conf_set_member_integer(location, &offset_str, resp->offset, 0); + loc_i = 0; + + if (have_offset) { + nxt_conf_set_member_integer(location, &offset_str, + resp->offset, loc_i++); - if (resp->line != 0) { - nxt_conf_set_member_integer(location, &line_str, - resp->line, 1); + if (resp->line != 0) { + nxt_conf_set_member_integer(location, &line_str, + resp->line, loc_i++); - nxt_conf_set_member_integer(location, &column_str, - resp->column, 2); + nxt_conf_set_member_integer(location, &column_str, + resp->column, loc_i++); + } + } + + if (have_pointer) { + nxt_conf_set_member_string(location, &path_str, + &resp->pointer, loc_i++); } } + + if (resp->suggestion.length != 0) { + n++; + + nxt_conf_set_member_string(value, &suggestion_str, + &resp->suggestion, n); + } } nxt_memzero(&pretty, sizeof(nxt_conf_json_pretty_t)); diff --git a/test/test_config_validation_errors.py b/test/test_config_validation_errors.py new file mode 100644 index 000000000..56651486f --- /dev/null +++ b/test/test_config_validation_errors.py @@ -0,0 +1,212 @@ +"""Tests for debuggable config validation errors. + +Covers the additive response fields introduced by roadmap item D5: + - ``location.path`` (RFC 6901 JSON Pointer to the offending member). + - ``suggestion`` (best-match member name when the input was a close typo). + +Legacy response fields (``error``, ``detail``, ``location.offset/line/column``) +must remain unchanged; several tests guard that. + +All cases are expressed using listeners + routes so they run without any +language SAPI module configured. +""" + +from unit.control import Control + +client = Control() + + +def _put(conf, url='/config'): + return client.conf(conf, url) + + +def test_unknown_top_level_key_has_root_path(): + r = _put({"foo": 1}) + assert 'error' in r + assert 'foo' in r['detail'] + assert r['location']['path'] == '' + + +def test_misspelled_listeners_suggests_listeners(): + r = _put({"listners": {}}) + assert 'error' in r + assert r.get('suggestion') == 'listeners' + + +def test_misspelled_applications_suggests_applications(): + r = _put({"aplications": {}}) + assert 'error' in r + assert r.get('suggestion') == 'applications' + + +def test_nested_unknown_key_has_container_path(): + r = _put( + { + "listeners": { + "*:8080": {"pass": "routes", "unknownkey": 1} + }, + "routes": [{"action": {"return": 200}}], + } + ) + assert 'error' in r + assert r['location']['path'] == '/listeners/*:8080' + + +def test_array_element_path_points_into_routes(): + r = _put( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"action": {"return": 200}}, + {"action": {"return": 200}}, + {"action": {"return": 200}, "unknownkey": 1}, + ], + } + ) + assert 'error' in r + assert r['location']['path'].startswith('/routes/2') + + +def test_type_error_carries_path(): + r = _put( + { + "listeners": {"*:8080": {"pass": 123}}, + "routes": [{"action": {"return": 200}}], + } + ) + assert 'error' in r + assert '/listeners' in r['location']['path'] + + +def test_rfc6901_escapes_tilde(): + """Route name ``a~b`` must encode as ``a~0b`` in the JSON Pointer.""" + r = _put( + { + "listeners": {"*:8080": {"pass": "routes/a~b"}}, + "routes": { + "a~b": [{"action": {"return": 200}, "unknownkey": 1}] + }, + } + ) + assert 'error' in r + assert r['location']['path'] == '/routes/a~0b/0' + + +def test_rfc6901_escapes_slash(): + """Route name ``a/b`` must encode as ``a~1b``, not split the pointer.""" + r = _put( + {"routes": {"a/b": [{"action": {"return": 200}, "unknownkey": 1}]}} + ) + assert 'error' in r + assert r['location']['path'] == '/routes/a~1b/0' + + +def test_parse_error_keeps_legacy_location_shape(): + # A malformed document never reaches validation, so it has no pointer to + # report; the location must stay exactly what it was before this change. + r = client.conf('{"listeners"', '/config') + assert 'error' in r + assert 'offset' in r['location'] + assert 'path' not in r['location'] + + +def test_no_suggestion_when_distance_exceeds_threshold(): + r = _put({"zzzzzzz": 1}) + assert 'error' in r + assert 'suggestion' not in r + + +def test_no_suggestion_for_short_unrelated_name(): + # The distance threshold alone admits 2 edits for a two-byte name, which + # is every two-letter member in the schema. A suggestion must also cover + # at most half the shorter name, so "zz" gets none rather than "if". + r = _put({"access_log": {"zz": 1}}) + assert 'error' in r + assert 'zz' in r['detail'] + assert 'suggestion' not in r + + +def test_app_type_errors_point_at_the_type_member(): + # nxt_conf_vldt_app() reaches into the "type" member itself rather than + # descending through the instrumented iterator, so all three of its + # checks have to report against "type" and not against the application + # object that contains it. + r = _put({"applications": {"demo": {"type": 1}}}) + assert 'error' in r + assert r['location']['path'] == '/applications/demo/type' + + r = _put({"applications": {"demo": {}}}) + assert 'error' in r + assert r['location']['path'] == '/applications/demo/type' + + r = _put({"applications": {"demo": {"type": "nosuch"}}}) + assert 'error' in r + assert r['location']['path'] == '/applications/demo/type' + + +def test_upstream_missing_servers_points_at_the_member(): + # Same convention as a missing NXT_CONF_VLDT_REQUIRED member: report + # against the absent member, not its container. + r = _put({"upstreams": {"u1": {}}}) + assert 'error' in r + assert r['location']['path'] == '/upstreams/u1/servers' + + +def _app(processes): + return { + "applications": { + "d": { + "type": "external", + "executable": "x", + "processes": processes, + } + } + } + + +def test_post_traversal_checks_point_at_the_member(): + # Semantic checks that run after nxt_conf_vldt_object() has finished the + # traversal see a path with the member's segment already popped, so they + # have to re-attribute explicitly or they name the container instead. + r = _put({"access_log": {"path": ""}}) + assert 'error' in r + assert r['location']['path'] == '/access_log/path' + + r = _put({"access_log": {"path": "/tmp/a", "format": "`{"}}) + assert 'error' in r + assert r['location']['path'] == '/access_log/format' + + for processes, member in [ + ({"spare": -1}, 'spare'), + ({"max": 0}, 'max'), + ({"spare": 5, "max": 2}, 'spare'), + ({"idle_timeout": -1}, 'idle_timeout'), + ]: + r = _put(_app(processes)) + assert 'error' in r, processes + assert ( + r['location']['path'] == f'/applications/d/processes/{member}' + ), processes + + +def test_backward_compat_success_has_no_new_fields(): + r = _put( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 200}}], + "applications": {}, + } + ) + assert 'success' in r + assert 'error' not in r + assert 'suggestion' not in r + + +def test_backward_compat_error_shape_preserved(): + r = _put({"foo": 1}) + assert 'error' in r + assert 'detail' in r + # 'success' key must never coexist with an error. + assert 'success' not in r + # Legacy message wording preserved verbatim. + assert r['detail'] == 'Unknown parameter "foo".' diff --git a/test/test_tls_conf_command.py b/test/test_tls_conf_command.py index 5a9a3f32c..555ec91d7 100644 --- a/test/test_tls_conf_command.py +++ b/test/test_tls_conf_command.py @@ -117,3 +117,26 @@ def check_conf_commands(conf_commands): check_conf_commands({"protocol": {}}) check_conf_commands({"protocol": "blah"}) check_conf_commands({"protocol": "TLSv1.2", "blah": ""}) + + +def test_tls_conf_command_invalid_path(skip_alert): + skip_alert(r'SSL_CONF_cmd', r'failed to apply new conf') + + # nxt_conf_vldt_object_conf_commands() iterates the object itself instead + # of going through the instrumented iterator, so the command name has to + # be pushed onto the validation path explicitly; otherwise the reported + # location stops at conf_commands and does not name the failing command. + r = client.conf( + {"certificate": "default", "conf_commands": {"protocol": {}}}, + 'listeners/*:8080/tls', + ) + + assert 'error' in r + + # builds without SSL_CONF_cmd reject the member itself, so there is no + # command name to attribute the error to. The validator's text is in + # "detail"; "error" is the generic "Invalid configuration." title. + if 'built without' in r['detail']: + pytest.skip('built without conf_commands support') + + assert r['location']['path'].endswith('/tls/conf_commands/protocol') From 7bea6d55dee53615a269694eb7275e2db0ea9774 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 18 Aug 2026 15:23:23 +0200 Subject: [PATCH 013/391] test: cover the application-side libunit port/fd lifecycle in every runtime The libunit port/fd lifecycle -- nxt_unit_add_port() and the add_port / remove_port callbacks, nxt_unit_port_release()'s destroy path, and nxt_unit_process_msg()'s `done:` descriptor cleanup -- lives in the application process, and every runtime links the same src/nxt_unit.c. Before this the suite drove those paths deliberately in exactly one runtime: test_go_application.py's two churn tests, added alongside the Go port-fd double-close fix. Everywhere else the coverage was incidental -- the conftest `run` fixture calling Log.check_alerts() after tests that do little more than send a request and check the body -- so an app-side double close() could reach CI as nothing but an intermittent "[alert]" line in one runtime's log. test_app_lifecycle.py churns those paths on purpose (worker count up and down, /control/.../restart storms, whole-application removal and re-add, requests in flight across all of it) and leans on the harness's existing teardown assertions to fail the run. It is parametrized over every runtime and self-skips the ones a leg did not build. Because it is runtime-agnostic, build-test.yml has to name it explicitly: the test_* glob would otherwise leave this covered in the python leg only. In sanitize.yml it joins the unit and php legs, where the same instrumentation is linked into the application process, so a double close(), a use-after-free of an nxt_unit_port_t or a stale descriptor becomes a hard failure instead of an intermittent alert. Also adds a go leg there. Go is the runtime whose app process links the sanitized libunit.a through cgo, and the only module registering libunit add_port / remove_port callbacks -- i.e. the only one exercising the ownership handoff those callbacks create. cgo + ASan is a new combination for this repo, so the leg is marked `experimental: true` and made non-blocking via continue-on-error until it has some green runs; drop the flag once it is stable. The Go package is consumed from build/go/src/unit.nginx.org/go, so the leg needs `make go-install` before the test apps will build. Verified locally on the php leg under an ASan+UBSan build: 5 passed, 40 skipped (the runtimes this tree does not build). The go leg is unverified locally -- only the php module builds here -- which is what `experimental` records. Three CI failures on the first run, all worth recording because each was a wrong assumption rather than a typo. The worker probe was blind to half the runtimes it claims to cover. It matched the process title `unit: "" application`, which nxt_process_title() sets in the forked child (src/nxt_process.c). For a `"type": "external"` application -- Go, Node, and the libunit test app -- nxt_external_start() then execve()s the target binary (src/nxt_external.c), replacing argv[] and the title with it. ps shows the worker as its own executable, the probe returned an empty set, and the two pid-dependent tests failed `assert 0 == 2` on every go and node leg while every embedded leg passed. The pass/fail split across the matrix was exactly embedded versus external, which is also why php-only local verification saw nothing. Workers are now taken from the PROTOTYPE's children. The prototype does not execve, so `unit: "" prototype` (src/nxt_main_process.c) survives for every runtime, and ppid is read from ps rather than via --ppid, which is procps-specific. Verified against an actual external application locally: with the old probe the two tests fail `assert 0 == 2` on the libunit app, and with this one they pass -- in 2.4s rather than 67.6s, since the old probe burned its full timeout on every call. halt_on_error=1 is dropped from UBSAN_OPTIONS. It is inert inside unitd, which is why it looked free, but an exec'd application process starts a fresh sanitizer runtime with the options intact -- so it was live exactly where the new go leg put it, and it aborts the worker on the FIRST violation including ones the allowlist waives, because that list is a post-hoc log scan and cannot stop a runtime abort. The known nxt_cpymem() zero-length copy (issue #200) killed the Go app mid-response and failed two unrelated tests with a missing response header. Unwaived reports are failed by the gate step, which is the mechanism that actually carries the weight. continue-on-error moves from the job to the steps. At job level it only stops the workflow RUN from failing; the job keeps its own failure conclusion, GitHub creates one check run per matrix job mirroring it, and the check still shows red in the PR rollup -- so the experimental go leg was blocking in exactly the way the flag was meant to prevent. The descriptor check counted the wrong thing. It compared the size of /proc//fd before and after a burst of requests, which cannot separate a libunit leak from the language runtime opening something of its own -- and some runtimes do: java-26 failed on a single descriptor the JVM kept, while java-17, 21 and 25 passed the same test. Polling for it to settle did not help, because it never went away. count_fds() now counts only the kinds of descriptor the test's own docstring claims to watch: sockets, shared memory segments, and regular files under the test's temp dir -- ports, incoming mmap segments and request spool files. Whatever a JVM opens for itself is out of scope, and a leaked port, segment or spool file still fails. Measured on a live php worker it counts 11 of the 15 open descriptors, so it is narrower without being empty. continue-on-error now covers every step an experimental leg can fail in, not just the test run and the gate. A cgo/ASan incompatibility would surface in `./configure go`, `make go` or `make go-install` -- all of them squarely on the unverified path -- and a failure there still failed the job, so the leg was blocking in exactly the case it was marked experimental for. Steps before those (apt, the core unitd build) keep no flag: they are shared infrastructure, and a failure there is a real problem on every leg rather than something this runtime is allowed to have. Deriving workers from the prototype fixed the external runtimes but left the teardown assertions able to pass for the wrong reason. If a removal or a restart takes the prototype down before its children, those children are reparented and pids() reports an empty set while the old generation is still running -- so "workers gone" would hold on an absent prototype rather than on absent workers, and a leaked generation would be indistinguishable from a clean restart. alive()/wait_for_gone() ask about specific pids instead, which no prototype ordering can fake. The reconfigure storm now records the workers serving immediately before it clears the configuration and waits for those to die, keeping wait_for_pids(0) as the separate check that nothing replaced them; the restart test waits for the previous generation to exit rather than settling for pids that merely differ. A pid the kernel recycled onto an unrelated process would read as alive. Over the seconds these tests span that is remote, and it fails an assertion rather than passing one -- the safe direction. Verified on php and libunit, embedded and external, 10 of 10. The prototype-first ordering itself could not be forced locally: teardown here completes before the check runs, so wait_for_gone() returns immediately. The assertion is prototype-independent by construction rather than by demonstration, which is the honest claim for it. A pedantic pre-push review found five more, three of them real defects. The keep-alive header dict was shared and mutable. HTTP1.http() writes Content-Length into the dict it is handed (test/unit/http.py), and the module passed its own KEEPALIVE constant, so the first mirrored POST left Content-Length in it permanently and every later keep-alive GET advertised a body it never sent -- hanging until the read timeout and failing on a missing status. CI hides this because each leg builds one language; a local run over several does not. A copy is passed instead. The restart test sampled the new generation before the old one was gone. nxt_router_app_restart_handler() sends QUIT to the old prototype and replies RPC_READY_LAST in the same handler, so restart() returns with the old prototype and its workers normally still alive. wait_for_pids(2) returns the first set of two it sees, which can be the two OLD workers, or one of each while both prototypes coexist -- and the disjointness assertion then fails a healthy restart, five times per runtime, most likely exactly where teardown is slowest. The wait for the old generation now comes first; disjointness stays as the pid-reuse guard. The libunit runtime never ran anywhere. build/unit_app_test is built only by the `tests` target, and both workflows run the default one, so the parameter that needs no language toolchain and links libunit directly -- the one with least between the test and the port/fd paths -- self-skipped in every leg while the legs reported green. Both workflows now build it, and sanitize configures --tests to be able to. Two smaller ones. The churn test asserted nothing about the pool it churns: conf_processes() returns on configuration success, not on spawn, so a run in which no worker was ever spawned looked identical to one in which all were. The un-awaited churn is deliberate -- racing spawn against teardown is the point -- so it keeps that and gains one awaited cycle in front of it, with the docstring saying which is which instead of claiming a handshake count it does not check. And a leg where every test skips exits 0 with clean logs, which is how a module that failed to build would report green; the run now fails if nothing passed, non-blocking on an experimental leg. The descriptor check measured one window, which cannot tell a leak from lazy allocation, and the app process does allocate lazily -- a further incoming mmap segment or an additional router port arrives when it is first needed, not during the warm-up. It failed for a single descriptor on java-26, narrowing the counter to sockets/segments/spool files did not settle it, and it then failed the same way on python for a descriptor the narrowed counter does watch. Both were growth that never repeated. A leak is growth that keeps happening, so the claim is now about the second of two identical bursts: whatever the first settles at, the second must not add to it. One-time lazy allocation lands in the first window and is tolerated by construction rather than by a threshold, and a per-request leak still shows up as roughly one descriptor per request in the second. The failure message lists what the process actually has open, so a report names the descriptor instead of only counting it. The empty-run guard was satisfied by tests belonging to other runtimes. Building build/unit_app_test to fix the silently-skipping libunit parameter also made those five parameters pass in every leg, so "N passed" held even when the leg's own module failed to build and every one of its tests skipped -- the exact condition the guard was added to catch, defeated by the fix above it. It now requires a passing test parametrized with the runtime the leg is named for. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014og6SESdCDxXzCpR6H9X33 --- .github/workflows/build-test.yml | 17 +- .github/workflows/sanitize.yml | 128 ++++- test/test_app_lifecycle.py | 801 +++++++++++++++++++++++++++++++ 3 files changed, 938 insertions(+), 8 deletions(-) create mode 100644 test/test_app_lifecycle.py diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 15be67055..a147c4d13 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -76,7 +76,12 @@ jobs: IFS='-' read -r module version <<< "${{ matrix.build }}" fi - testpath="test/test_${module}*" + # test_app_lifecycle.py is runtime-agnostic: it is parametrized over + # every runtime and self-skips the ones this leg did not build, so + # each language leg must be told about it explicitly -- the + # test_* glob below would otherwise leave the app-process + # libunit port/fd lifecycle covered in the python leg only. + testpath="test/test_${module}* test/test_app_lifecycle.py" # Run all tests for "unit" and "python" # Python is the default module for tests @@ -219,6 +224,16 @@ jobs: run: | make -j4 -k || make + # The default target is `all: unitd manpage`, so --tests above only + # makes this target available -- it does not build it. Without it + # build/unit_app_test does not exist and test_app_lifecycle.py's + # `libunit` parameter self-skips in every leg, silently: the runtime + # that needs no language toolchain and links libunit directly, i.e. + # the one that exercises the port/fd lifecycle with the least in the + # way, would contribute nothing while the legs still reported green. + - name: Make the libunit test application + run: make -j4 build/unit_app_test + ## ## Go ## diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index eb19570d2..2a5aee3b1 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -58,6 +58,18 @@ jobs: name: "sanitize (${{ matrix.runtime }})" runs-on: ubuntu-latest + # NOTE: `continue-on-error` is deliberately NOT set at job level here. + # It is set on every step an experimental leg can fail in: the + # runtime-specific configure/make steps, the Go package install, the test + # run and the gate. Anything earlier -- apt, the core unitd build -- is + # shared infrastructure, and a failure there is a real problem on every + # leg rather than something the experimental runtime is allowed to have. + # Job-level continue-on-error only stops the WORKFLOW RUN from failing; the + # job keeps its own `failure` conclusion, GitHub creates one check run per + # matrix job mirroring it, and the check still shows red in the PR rollup. + # To make an experimental leg genuinely non-blocking the flag has to be on + # the steps that can fail -- see the run and gate steps below. + # Blocking on purpose: this job exists to red-line lifecycle teardown # regressions. Its first run caught the controller's raw c->link unlinks # desyncing c->idle from the tracking queues, crashing every --debug build @@ -100,6 +112,16 @@ jobs: # buffer reaches the parser's recycle path. Without ASan a stale # buffer is usually silently reused, so only this leg turns that # back into a hard failure. + # + # test_app_lifecycle drives the APPLICATION process's libunit + # port/fd lifecycle (process-count churn, control restart storms, + # whole-application remove/re-add, churn under keep-alive). The + # language module is linked into the app process with the same + # instrumentation as unitd, so a double close(), a use-after-free + # of an nxt_unit_port_t or a stale descriptor in libunit becomes a + # hard ASan failure here instead of an intermittent "[alert]" line + # in one runtime's log. It self-skips every runtime that is not + # built into this leg's unitd. tests: >- test/test_graceful_reload.py test/test_listener_drain.py @@ -108,6 +130,7 @@ jobs: test/test_proxy.py test/test_chunked.py test/test_conn_recycle.py + test/test_app_lifecycle.py - runtime: php apt: "" configure: php @@ -116,7 +139,29 @@ jobs: # and --restart's teardown rmtree trips on test_php_application_ # forbidden's restricted-permission fixture (rmtree PermissionError). pytest_opts: "" - tests: test/test_php_application.py + tests: >- + test/test_php_application.py + test/test_app_lifecycle.py + # Go is where the app-side port-fd double-close actually manifested + # (a flaky "close(N) failed: Bad file descriptor" in the go isolation + # leg) and it is the only module that registers libunit add_port / + # remove_port callbacks, i.e. the only one that exercises the + # ownership handoff those callbacks create. Running its churn tests + # under ASan turns a silently-reused descriptor into a hard failure. + # + # experimental: cgo + ASan is a new combination for this repo (the Go + # binary links the sanitized libunit.a through cgo). Kept + # non-blocking for its first green runs; drop `experimental` once it + # is stable. + - runtime: go + apt: "" + configure: go --go-path= + make: go + pytest_opts: "" + experimental: true + tests: >- + test/test_go_application.py + test/test_app_lifecycle.py steps: - uses: actions/checkout@v5 @@ -172,6 +217,7 @@ jobs: run: | ./configure \ --debug \ + --tests \ --openssl \ --cc-opt="-fsanitize=address,undefined -fno-omit-frame-pointer -O1" \ --ld-opt="-fsanitize=address,undefined" @@ -179,28 +225,58 @@ jobs: - name: Make unit run: make -j$(nproc) unitd + # --tests above only makes this target available; the default target + # does not build it. Without it the `libunit` parameter of + # test_app_lifecycle.py self-skips, and the runtime that links libunit + # directly -- the one worth the most under a sanitizer, since nothing + # else sits between the test and the port/fd paths -- would be absent + # from precisely the legs built to instrument them. + - name: Make the libunit test application + run: make -j$(nproc) build/unit_app_test + # Same probe-leak rationale as the core configure step above. - name: Configure ${{ matrix.runtime }} module + continue-on-error: ${{ matrix.experimental == true }} env: ASAN_OPTIONS: "detect_leaks=0" run: ./configure ${{ matrix.configure }} - name: Make ${{ matrix.runtime }} module + continue-on-error: ${{ matrix.experimental == true }} run: make -j$(nproc) ${{ matrix.make }} + # The Go package is consumed from build/go/src/unit.nginx.org/go by + # test/unit/applications/lang/go.py (it writes a go.mod `replace` + # pointing there), so the test apps only build after go-install. The + # generated env.go carries the tree's --cc-opt/--ld-opt into cgo, which + # is how the sanitized libunit.a reaches the Go app process. + - name: Install Go package + continue-on-error: ${{ matrix.experimental == true }} + if: matrix.runtime == 'go' + run: make go-install + # ASAN_OPTIONS / UBSAN_OPTIONS are step-level env so pytest and every # forked unitd process inherit them. # detect_leaks=0 # the fork-heavy daemon makes LeakSanitizer extremely noisy; turning # leak detection on is tracked as a follow-up, once the teardown # paths are use-after-free / null-deref clean. - # print_stacktrace / halt_on_error - # honoured wherever the options string survives. They do NOT take + # print_stacktrace + # honoured wherever the options string survives. It does NOT take # effect inside unitd itself -- see the header of # .github/scripts/check-sanitizer-reports.sh -- which is exactly why # the gate below reads unit.log instead of trusting a log_path - # directory. Kept because they cost nothing and do apply to any - # helper binary the suite runs. + # directory. Kept because it costs nothing and does apply to the + # application processes a `"type": "external"` module execve()s, + # which start a fresh sanitizer runtime with the options intact. + # halt_on_error is deliberately NOT set, for that same reason: in an + # exec'd app process it is live, and it aborts the worker on the + # FIRST violation -- including ones the allowlist waives, since that + # list is a post-hoc log scan and cannot stop a runtime abort. With + # it on, the known nxt_cpymem() zero-length copy (issue #200) killed + # the Go app mid-response and failed unrelated tests with a missing + # response header. Unwaived reports are failed by the gate step + # below, which is the mechanism that is actually load-bearing. # # --save-log keeps /unit.log after each test. Without it # test/conftest.py removes the per-restart temp dir, and with it the only @@ -208,11 +284,48 @@ jobs: # below scans. pytest_opts is per-runtime (e.g. --restart for the # graceful tests). - name: Run ${{ matrix.runtime }} sanitizer subset + continue-on-error: ${{ matrix.experimental == true }} env: ASAN_OPTIONS: "detect_leaks=0" - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" + UBSAN_OPTIONS: "print_stacktrace=1" + run: | + set -o pipefail + python3 -m pytest -v --save-log ${{ matrix.pytest_opts }} \ + ${{ matrix.tests }} | tee "${RUNNER_TEMP}/pytest.out" + + # A leg in which every test skipped exits 0, writes clean logs, and is + # therefore green -- indistinguishable from one that ran and passed. + # That is how a module which failed to build hides: check_*() marks the + # runtime unavailable at session start, every parametrized test skips, + # and nothing anywhere says so. It matters most for a leg being + # watched for "first green runs" before dropping `experimental`, where + # a skipped-everything green is precisely the wrong signal. + - name: Fail on an empty test run + if: always() + continue-on-error: ${{ matrix.experimental == true }} run: | - python3 -m pytest -v --save-log ${{ matrix.pytest_opts }} ${{ matrix.tests }} + if [ ! -s "${RUNNER_TEMP}/pytest.out" ]; then + echo "::error::The ${{ matrix.runtime }} leg produced no pytest" \ + "output; the run step did not get that far." + exit 1 + fi + + # The pass has to belong to THIS runtime. A bare "N passed" is + # satisfied by the runtime-agnostic parameters that run in every + # leg -- libunit above all, which needs no language toolchain and + # so passes even when this leg's own module failed to build and + # every one of its tests skipped. That is the precise condition + # this guard exists to catch, so it looks for a passing test + # parametrized with the runtime's own name. + if ! grep -qE "\[${{ matrix.runtime }}\].*PASSED" \ + "${RUNNER_TEMP}/pytest.out"; then + echo "::error::No ${{ matrix.runtime }} test passed in the" \ + "${{ matrix.runtime }} leg. Its tests skipped or errored," \ + "which exits 0 and would otherwise report this leg green" \ + "while covering nothing of the runtime it is named for." + tail -30 "${RUNNER_TEMP}/pytest.out" + exit 1 + fi - name: Upload unit logs if: always() @@ -234,6 +347,7 @@ jobs: # server". Fail the job explicitly on what the logs actually contain. - name: Fail on sanitizer reports if: always() + continue-on-error: ${{ matrix.experimental == true }} run: | .github/scripts/check-sanitizer-reports.sh \ "${TMPDIR:-/tmp}" .github/sanitizer-allowlist.txt diff --git a/test/test_app_lifecycle.py b/test/test_app_lifecycle.py new file mode 100644 index 000000000..313b613c8 --- /dev/null +++ b/test/test_app_lifecycle.py @@ -0,0 +1,801 @@ +"""Application-process lifecycle churn, parametrized over every runtime. + +Rationale +--------- +The libunit port/fd lifecycle -- nxt_unit_add_port() and the add_port / +remove_port callbacks, nxt_unit_port_release()'s destroy path, and +nxt_unit_process_msg()'s `done:` descriptor cleanup -- lives in the +*application* process, and every runtime links the same src/nxt_unit.c. A bug +there (a double close(), a leaked descriptor, a use-after-free of a port +struct) surfaces as an "[alert]" line written by the app process into unit.log +during port teardown, or as a slowly growing /proc//fd. + +Before this module the suite exercised those paths deliberately in exactly one +runtime -- test_go_application.py's two churn tests, added alongside the Go +port-fd double-close fix. Every other runtime got only incidental coverage +from the conftest `run` fixture calling Log.check_alerts() after tests that do +little more than "send a request, check the body", so an app-side double close +could reach CI as nothing but an intermittent alert in one runtime's log. + +These tests deliberately churn the app-side port/fd lifecycle -- worker count +up and down, /control/.../restart storms, whole-application removal and re-add, +requests in flight across all of it -- and lean on the harness's existing +teardown assertions to fail the run: + + * conftest `run` fixture -> Log.check_alerts(): any "[alert]" from the app + process (e.g. "close(N) failed: Bad file descriptor") fails the test, and + any "Sanitizer" line fails it in the ASan/UBSan legs. + * conftest `_check_fds()`: descriptor leaks in main/router/controller + (with --fds-threshold=0, the CI default). + +plus one assertion the harness makes nowhere else: + + * test_app_lifecycle_app_fds_stable(): descriptor growth inside the + *application* process itself. + +Runtime coverage is table-driven (RUNTIMES below); a runtime that is not built +into this unitd self-skips, so the same file can run in every per-language CI +leg with only that leg's parameter executing. Adding a runtime is one entry. + +CI wiring note: .github/workflows/build-test.yml derives a per-module test glob +(test/test_*), so this file is named explicitly in that job's +`testpath` -- otherwise only the python/unit legs would ever run it. +""" + +import os +import subprocess +import time +from pathlib import Path + +import pytest + +from unit.applications.lang.go import ApplicationGo +from unit.applications.lang.java import ApplicationJava +from unit.applications.lang.node import ApplicationNode +from unit.applications.lang.perl import ApplicationPerl +from unit.applications.lang.php import ApplicationPHP +from unit.applications.lang.python import ApplicationPython +from unit.applications.lang.ruby import ApplicationRuby +from unit.applications.lang.wasm_component import ApplicationWasmComponent +from unit.applications.proto import ApplicationProto +from unit.option import option + +# Deliberately no module-level `client`: conftest.pytest_generate_tests() keys +# on it to fan a module out over language versions, which would collide with +# the per-runtime parametrization below. Deliberately no module-level +# `prerequisites` either: availability is per parameter, checked in `app`. + + +class ApplicationLibunit(ApplicationProto): + """The bare C libunit app, src/test/nxt_unit_app_test.c. + + Needs no language toolchain -- `./configure --tests && make tests` builds + it -- and links libunit directly, so it exercises the port/fd lifecycle + under test with nothing else in the way. It answers with a "Request data:" + dump that embeds the request body rather than mirroring it verbatim, hence + mirror=False in the table below. + """ + + application_type = 'external' + + @staticmethod + def executable(): + return f'{option.current_dir}/build/unit_app_test' + + def load(self, script, **kwargs): + self._load_conf( + { + "listeners": {"*:8080": {"pass": f"applications/{script}"}}, + "applications": { + script: { + "type": "external", + "processes": {"spare": 0}, + "working_directory": option.temp_dir, + "executable": ApplicationLibunit.executable(), + }, + }, + }, + **kwargs, + ) + + +# runtime -> (client factory, app name, required modules, required features, +# body is mirrored back verbatim, +# answers a body larger than one shared-memory segment) +# +# The last flag is false only for the libunit sample: it sizes its response +# buffer from the whole request buffer (src/test/nxt_unit_app_test.c), and +# nxt_unit_response_buf_alloc() refuses anything above PORT_MMAP_DATA_SIZE, so +# a request that fills a segment leaves it unable to build a response at all. +# That is a property of the sample app, not of the port handling under test. +RUNTIMES = { + 'libunit': (ApplicationLibunit, 'libunit', [], [], False, False), + 'go': (ApplicationGo, 'mirror', ['go'], [], True, True), + 'java': (ApplicationJava, 'mirror', ['java'], [], True, True), + 'node': (ApplicationNode, 'mirror', ['node'], [], True, True), + # there is no test/perl/mirror; body_array answers a fixed 10-byte body, + # which is enough to drive GET traffic through the churn + 'perl': (ApplicationPerl, 'body_array', ['perl'], [], False, True), + 'php': (ApplicationPHP, 'mirror', ['php'], [], True, True), + 'python': (ApplicationPython, 'mirror', ['python'], [], True, True), + 'ruby': (ApplicationRuby, 'mirror', ['ruby'], [], True, True), + 'wasm-wasi-component': ( + ApplicationWasmComponent, + 'hello_world', + ['wasm-wasi-component'], + ['cargo_component'], + False, + True, + ), +} + +BODY = '0123456789' * 200 + +# PORT_MMAP_DATA_SIZE (src/nxt_port_memory_int.h): the payload area of one +# shared-memory segment, and therefore the amount of request data the router +# can hand an application before it has to create another one. +SEGMENT = 10 * 1024 * 1024 + +KEEPALIVE = {'Host': 'localhost', 'Connection': 'keep-alive'} + + +class LifecycleApp: + """Ties a runtime's client to its app name and request shape.""" + + def __init__(self, runtime, client, name, mirror, oversized): + self.runtime = runtime + self.client = client + self.name = name + self.mirror = mirror + self.oversized = oversized + + def load(self, **kwargs): + self.client.load(self.name, **kwargs) + + def conf_processes(self, conf): + assert 'success' in self.client.conf( + conf, f'applications/{self.name}/processes' + ), f'{self.runtime}: configure processes' + + def restart(self): + assert 'success' in self.client.conf_get( + f'/control/applications/{self.name}/restart' + ), f'{self.runtime}: restart' + + def request(self, sock=None, keepalive=False, **kwargs): + """One request that forces real request/response traffic. + + POST with a body where the runtime mirrors it back -- bodies travel + through the shared-memory and spool-file paths that carry descriptors + to the app -- plain GET otherwise. Returns (response, socket); the + socket is None unless the connection was kept alive. + """ + if keepalive or sock is not None: + # A COPY. HTTP1.http() writes Content-Length into the dict it + # is handed (test/unit/http.py), so passing the module-level + # object would leave that header in it for good: the first + # mirrored POST poisons every later keep-alive GET, which then + # advertises a body it never sends and hangs until the read + # timeout. Only one language is built per CI leg, so today this + # needs two runtimes in one session to bite -- which is exactly + # what a local full run is. + kwargs['headers'] = dict(KEEPALIVE) + kwargs['start'] = True + # the server will not close a keep-alive connection, so recvall() + # must be allowed to stop on a short timeout instead of on EOF. + # + # Every keep-alive read therefore costs this timeout in full, and + # the same timeout is all that separates "the response is + # complete" from "the app has not answered yet". Two seconds + # buys margin over an ASan-instrumented worker on a loaded runner + # without paying much for it; the startup race that would need + # more than that is closed by waiting for the workers instead. + kwargs['read_timeout'] = 2 + + if sock is not None: + kwargs['sock'] = sock + + if self.mirror: + out = self.client.post(body=BODY, **kwargs) + else: + out = self.client.get(**kwargs) + + resp, sock = out if isinstance(out, tuple) else (out, None) + + # recvall() returns b'' rather than failing when a non-default + # read_timeout expires, and an empty response parses to {} -- which + # would surface as a KeyError instead of naming the runtime that went + # quiet. + assert 'status' in resp, f'{self.runtime}: no response before timeout' + + assert resp['status'] == 200, f'{self.runtime}: status' + + if self.mirror: + assert resp['body'] == BODY, f'{self.runtime}: mirror' + + return resp, sock + + def oversized_request(self, pool): + """One POST the router's `pool` existing segments cannot serve. + + nxt_router_prepare_msg() (src/nxt_router.c) copies the whole in-memory + body into the router's outgoing segments for this application, filling + a segment that has room and creating another when none has. A body of + `pool` whole segments is therefore always one chunk too large for the + `pool` segments already there -- the request line and fields took one + out of the first -- and forces exactly one to be created. + + The body has to reach the application through shared memory rather + than as a spool file, which is what the caller's body_buffer_size is + for; a spooled body travels as a descriptor and never touches the + segments. + + Nothing is asserted about the echoed body: the size comes from the + router's geometry, not from what a runtime is willing to hand back, + and the claim here is that the router created a segment, which the + caller checks directly. + """ + + resp = self.client.post( + body='x' * (pool * SEGMENT), read_buffer_size=1024 * 1024 + ) + + assert resp['status'] == 200, f'{self.runtime}: oversized status' + + def pids(self): + # Worker pids come from the PROTOTYPE's children, not from the + # workers' own process titles. + # + # `unit: "" application` is set by nxt_process_title() in the + # forked child (src/nxt_process.c), and for an embedded module that + # is the end of it. A `"type": "external"` application -- Go, Node, + # and the libunit test app -- then execve()s the target binary + # (src/nxt_external.c), which replaces argv[] wholesale, so the + # worker shows up in ps as its own executable and the title is gone. + # Matching on it finds nothing for exactly the runtimes this module + # most wants to cover. + # + # The prototype does not execve, so `unit: "" prototype` + # (built in src/nxt_main_process.c) survives for every runtime, and + # its children are the workers. ppid is read from ps rather than + # via `--ppid`, which is procps-specific. + time.sleep(0.2) + + output = subprocess.check_output( + ['ps', 'ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'args='] + ).decode() + + rows = [] + for line in output.splitlines(): + parts = line.split(None, 2) + + if len(parts) == 3: + rows.append(parts) + + marker = f'unit: "{self.name}" prototype' + prototypes = {pid for pid, _, args in rows if marker in args} + + if not prototypes: + return set() + + return {pid for pid, ppid, _ in rows if ppid in prototypes} + + def wait_for_pids(self, count, timeout=15): + # `timeout` is nominal: pids() sleeps 0.2s of its own per sample, so + # the real ceiling is about twice this. Left as-is rather than + # tightened, because the margin is what absorbs a slow sanitizer + # teardown; the name is the thing that lies, so it is documented. + for _ in range(timeout * 5): + pids = self.pids() + + if len(pids) == count: + return pids + + time.sleep(0.2) + + return self.pids() + + def alive(self, pids): + """Which of `pids` still exist, prototype or no prototype. + + pids() derives workers from the prototype's children, so a teardown + that removes the prototype BEFORE its children leaves those children + reparented and invisible to it -- and an absent prototype would then + read as "no workers left", which is exactly the conclusion the + teardown assertions must not reach by default. Asking about + specific pids keeps them honest: a worker that outlives its + prototype still answers here. + + A pid the kernel recycled onto an unrelated process would count as + alive. Over the seconds these tests span that is remote, and it + would fail an assertion rather than pass one, which is the safe + direction to be wrong in. + """ + + if not pids: + return set() + + output = subprocess.check_output(['ps', 'ax', '-o', 'pid=']).decode() + running = { + line.strip() for line in output.splitlines() if line.strip() + } + + return set(pids) & running + + def wait_for_gone(self, pids, timeout=15): + """Wait for every pid in `pids` to leave the process table.""" + + for _ in range(timeout * 5): + left = self.alive(pids) + + if not left: + return set() + + time.sleep(0.2) + + return self.alive(pids) + + +def _skip_reason(runtime): + _, _, modules, features, _, _ = RUNTIMES[runtime] + + if runtime == 'libunit': + # built only by `./configure --tests && make tests` + if not Path(ApplicationLibunit.executable()).is_file(): + return 'build/unit_app_test not built (configure --tests)' + + return None + + available = option.available['modules'] + missed = [m for m in modules if not available.get(m)] + if missed: + return f'no {", ".join(missed)} module(s)' + + available = option.available['features'] + missed = [f for f in features if not available.get(f)] + if missed: + return f'{", ".join(missed)} feature(s) not supported' + + return None + + +@pytest.fixture(params=list(RUNTIMES)) +def app(request): + runtime = request.param + + reason = _skip_reason(runtime) + if reason is not None: + pytest.skip(f'{runtime}: {reason}') + + factory, name, _, _, mirror, oversized = RUNTIMES[runtime] + + return LifecycleApp(runtime, factory(), name, mirror, oversized) + + +def count_fds(pid): + """Descriptors of the kinds libunit hands an application process. + + A raw count of /proc//fd cannot tell a libunit leak from the host + runtime opening something of its own, and some runtimes legitimately do: + a JVM may open a jar, a JIT artefact or an eventfd part-way through a run + and keep it. Counting everything made this test fail on java-26 for a + single descriptor while java-17, 21 and 25 passed -- a difference in the + JVM, not in the port handling this test exists to watch. + + So count only what the docstring of the test actually claims: sockets + (ports), shared memory segments (incoming mmap), and regular files under + the test's own temp dir (request spool files). Anything the language + runtime opens for its own purposes is deliberately out of scope, and a + real leak of a port, a segment or a spool file still fails. + """ + + fds = Path(f'/proc/{pid}/fd') + + if not fds.is_dir(): + return None + + n = 0 + + try: + for fd in fds.iterdir(): + try: + target = os.readlink(fd) + + except OSError: + # the descriptor closed underneath us; it is not a leak + continue + + if ( + target.startswith('socket:') + or target.startswith('/memfd:') + or target.startswith('/dev/shm/') + or target.startswith(f'{option.temp_dir}/') + ): + n += 1 + + except OSError: + return None + + return n + + +def fd_targets(pid): + """What the counted descriptors of `pid` point at, for a failure message. + + A bare count says growth happened; this says what grew, which is the + difference between a report someone can act on and one that starts an + investigation from nothing. + """ + + fds = Path(f'/proc/{pid}/fd') + + if not fds.is_dir(): + return set() + + targets = set() + + try: + for fd in fds.iterdir(): + try: + targets.add(os.readlink(fd)) + + except OSError: + continue + + except OSError: + return set() + + return targets + + +def shm_segments(pid): + """Shared-memory segments this process was handed by another one. + + The descriptor that carries a segment is the one the fd test is blind to + by default: nxt_unit_incoming_mmap() (src/nxt_unit.c) maps it and + nxt_unit_process_msg()'s `done:` closes it, so a healthy application keeps + the mapping and no descriptor -- which is exactly why a *leak* of that + descriptor is only visible in a window where a segment actually arrived. + Counting the mappings is how the test knows a window contained one. + + Read from /proc//maps, and narrowed three ways, because a process + maps shared memory for several unrelated reasons. Segment-sized, which + drops the port queues (a few pages each). Named `unit.`, which drops + whatever the language runtime maps for itself. Not named `unit..`, + which drops the segments this process created for its own responses -- + it opens and closes those itself (nxt_unit_new_mmap()), so they carry no + incoming descriptor to leak. + + Returns the segment names rather than a count, so a failure can say which + ones were there. + """ + + maps = Path(f'/proc/{pid}/maps') + + segments = set() + + try: + lines = maps.read_text().splitlines() + + except OSError: + return segments + + for line in lines: + parts = line.split(None, 5) + + if len(parts) < 6: + continue + + bounds, path = parts[0], parts[5] + + if not path.startswith('/memfd:') and not path.startswith('/dev/shm/'): + continue + + if f'unit.{pid}.' in path or 'unit.' not in path: + continue + + start, _, end = bounds.partition('-') + + if int(end, 16) - int(start, 16) < SEGMENT: + continue + + segments.add(path) + + return segments + + +def test_app_lifecycle_process_churn(app): + """Scale the worker pool up and down with traffic in between. + + A scale-up spawns workers that run the full libunit port handshake + (NEW_PORT carrying a socket and a queue descriptor, the add_port callback, + PORT_ACK); a scale-down tears them down (QUIT, REMOVE_PID, port destroy). + Generalization of test_go_application.py's + test_go_application_port_fd_churn to every runtime. + + The loop does NOT wait for each pool size to be reached, which is + deliberate: conf_processes() returns on configuration success, not on + spawn, so an un-awaited rewrite races the next teardown against workers + that are still starting -- the overlap this test exists to put under a + sanitizer. That also means a given iteration may spawn fewer than four + workers, so the count is not asserted per round. One awaited cycle runs + first, to establish that scaling works at all before the racing starts; + without it a run in which no worker was ever spawned would look the same + as a run in which every one of them was. + """ + app.load() + + app.conf_processes('4') + assert len(app.wait_for_pids(4)) == 4, f'{app.runtime}: scaled up to 4' + + app.conf_processes('1') + assert len(app.wait_for_pids(1)) == 1, f'{app.runtime}: scaled down to 1' + + for _ in range(10): + app.conf_processes('4') + app.request() + + app.conf_processes('1') + app.request() + + +def test_app_lifecycle_restart_control(app): + """/control/applications//restart storm. + + A control restart replaces every worker: the outgoing generation is sent + QUIT and drops its ports while the incoming one announces fresh ports to + the same router. The pid-rotation assertion keeps a no-op restart from + making this pass vacuously. + """ + app.load(processes=2) + + pids = app.wait_for_pids(2) + assert len(pids) == 2, f'{app.runtime}: two workers' + + for _ in range(5): + app.restart() + + # Wait for the OLD generation to go before sampling the new one. + # nxt_router_app_restart_handler() (src/nxt_router.c) sends QUIT to + # the old prototype and replies RPC_READY_LAST in the same handler, + # so restart() returns while the old prototype and its workers are + # normally still alive. Sampling first would take the first set of + # two it sees -- which can be the two OLD workers, or one of each + # while both prototypes coexist -- and the disjointness assertion + # below would then fail a perfectly healthy restart. Under the + # sanitizer, where teardown is slow, that is the likely outcome + # rather than the unlucky one. + assert ( + app.wait_for_gone(pids) == set() + ), f'{app.runtime}: workers from the previous generation still alive' + + new_pids = app.wait_for_pids(2) + + assert len(new_pids) == 2, f'{app.runtime}: two workers after restart' + + # Kept as the pid-reuse guard: with the old generation already gone, + # an overlap here would mean the kernel handed a new worker a pid we + # had just watched die. + assert not new_pids & pids, f'{app.runtime}: all workers replaced' + + app.request() + + pids = new_pids + + +def test_app_lifecycle_reconfigure_storm(app): + """Remove and re-add the whole application repeatedly. + + Clearing `applications` makes the router QUIT every worker and drop the + app's ports; the app process then runs nxt_unit_quit() -> + ctx_free/lib_release, which is where the port destroy path closes the + descriptors libunit owns. Re-adding runs the whole spawn/handshake again. + + The configuration is captured once and replayed, because several runtime + clients stage their application into the temp dir on load() and cannot + stage it twice. + """ + app.load(processes=2) + app.request() + + conf = app.client.conf_get('/config') + + for _ in range(5): + # Not a bare pids() snapshot. Starting the pool is asynchronous and + # app.request() only proves ONE worker is ready, so an unvalidated + # sample can hold one of the two while the second is still spawning. + # A worker missing from `before` is one wait_for_gone() will never + # wait for, and wait_for_pids(0) would then be satisfied by the + # absent prototype again -- putting the hole straight back. + # + # "processes": 2 here is a fixed count, not a spare/max range, so the + # expected number is exact. + before = app.wait_for_pids(2) + + assert len(before) == 2, ( + f'{app.runtime}: expected 2 workers before removal, ' + f'saw {sorted(before)}' + ) + + assert 'success' in app.client.conf( + {"listeners": {}, "applications": {}} + ), f'{app.runtime}: clear conf' + + # Both checks are needed. wait_for_gone() names the processes that + # were serving a moment ago and waits for those to die, which an + # early prototype exit cannot fake; wait_for_pids(0) then confirms + # nothing was started to replace them. + assert ( + app.wait_for_gone(before) == set() + ), f'{app.runtime}: pre-removal workers still alive' + + assert app.wait_for_pids(0) == set(), f'{app.runtime}: workers gone' + + assert 'success' in app.client.conf(conf), f'{app.runtime}: restore' + + app.request() + + +def test_app_lifecycle_churn_under_keepalive(app): + """Churn the worker pool while keep-alive connections stay open. + + Requests arriving on connections that outlive a worker generation are the + case where a router-side response port and an app-side port lifecycle + overlap -- the interleaving in which a stale descriptor is most likely to + be closed a second time. + """ + app.load(processes=2) + + # not decoration: the first keep-alive read is the one racing application + # startup, and it gives up after read_timeout with whatever it has rather + # than failing, so a worker that is not up yet reads as an empty response. + app.wait_for_pids(2) + + socks = [] + try: + for _ in range(3): + _, sock = app.request(keepalive=True) + socks.append(sock) + + for i in range(4): + app.conf_processes('4' if i % 2 else '1') + + for j, sock in enumerate(socks): + _, socks[j] = app.request(sock=sock) + + finally: + for sock in socks: + if sock is not None: + sock.close() + + +def test_app_lifecycle_app_fds_stable(app): + """No per-request descriptor growth inside the application process. + + conftest's _check_fds() only watches main/router/controller. The libunit + paths that hand descriptors to an app -- incoming mmap segments, request + spool files, port sockets -- are only observable here, so a leak of one + descriptor per request or per port announcement is invisible to the rest + of the suite. + + Measured across TWO identical bursts rather than before-and-after one. + A single burst cannot tell a leak from lazy one-time allocation, and the + app process does allocate lazily: a further incoming mmap segment or an + additional router port arrives when it is first needed, not during the + warm-up. Counting one window made this fail for a single descriptor on + java-26 and again on python, both times for growth that never repeated. + A leak is growth that keeps happening, so the claim is about the SECOND + window: whatever the first burst settles, the second must not add to it. + + Each burst ends with one oversized POST, and that is not decoration. The + router creates a segment only when the ones it has for this application + cannot serve a request, and 2000-byte requests release their chunks as + they complete, so a burst of them reuses the segment the warm-up caused + and never makes the router announce another. A descriptor leaked per + incoming segment would then already be inside BOTH counts and cancel out + -- the one leak this test names in its own docstring, invisible to it. + The oversized request forces an announcement; the burst asserts that it + happened, so the probe cannot quietly stop working. + + The POST grows by one segment per burst because the pool grows with it: + the segment the first burst forced is free again by the time the second + runs, and would swallow a repeat of the same body. What stays equal + between the two windows is the thing being counted -- one segment + created, one descriptor received -- which is what makes the difference + between them a leak rather than a workload difference. + + Runtimes whose application cannot answer such a request keep the bursts + and lose only the segment part of the claim (see RUNTIMES); driving one + into a response it cannot build would test the sample app, not libunit. + """ + app.load(processes=1) + + # The oversized body must stay in memory to travel through shared memory + # at all: past body_buffer_size the router spools it to a file and sends + # a descriptor instead (nxt_h1p_request_body_read(), src/nxt_h1proto.c). + # + # The margin over the largest body sent below is deliberate. Sitting on + # the boundary would make the probe depend on which side of it a spooling + # test is written, and the wasm component's read loop + # (src/wasm-wasi-component/src/lib.rs) does not terminate on a short read, + # so a body that spooled would hang that leg rather than fail it. + assert 'success' in app.client.conf( + { + "http": { + "body_buffer_size": 4 * SEGMENT, + "max_body_size": 4 * SEGMENT, + } + }, + 'settings', + ), f'{app.runtime}: buffer oversized bodies in memory' + + pids = app.wait_for_pids(1) + assert len(pids) == 1, f'{app.runtime}: single worker' + + pid = pids.pop() + + # warm-up: the first requests create the outgoing mmap segments and, in + # threaded runtimes, the per-thread contexts + for _ in range(10): + app.request() + + def burst(pool): + for _ in range(40): + app.request() + + if app.oversized: + app.oversized_request(pool) + + # let anything the burst opened and is about to close settle, so a + # descriptor in flight is not counted as retained + settled = count_fds(pid) + + for _ in range(50): + n = count_fds(pid) + + if n is None or n == settled: + break + + settled = n + time.sleep(0.1) + + return settled + + if count_fds(pid) is None: + pytest.skip('no /proc//fd on this platform') + + warm = shm_segments(pid) + + first = burst(1) + after_first = shm_segments(pid) + + second = burst(2) + after_second = shm_segments(pid) + + assert app.pids() == {pid}, f'{app.runtime}: same worker throughout' + + if app.oversized: + # Both windows, not just the second: a first burst that forced + # nothing would leave its own leak out of `first` and make the + # comparison below read a per-segment leak as one-time growth. + assert len(after_first) > len(warm), ( + f'{app.runtime}: no incoming segment in the first burst, so a ' + f'leak of one cannot be seen: {sorted(warm)} -> ' + f'{sorted(after_first)}' + ) + + assert len(after_second) > len(after_first), ( + f'{app.runtime}: no incoming segment in the second burst, so a ' + f'leak of one cannot be seen: {sorted(after_first)} -> ' + f'{sorted(after_second)}' + ) + + assert first is not None and second is not None, ( + f'{app.runtime}: worker vanished mid-measurement' + ) + + assert second - first <= option.fds_threshold, ( + f'{app.runtime}: descriptor leak in application process: ' + f'{first} -> {second} across a repeated identical burst; ' + f'open now: {sorted(fd_targets(pid))}' + ) From bb35d17d8eb958911f13b929e1f54620cc3019d3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 18 Aug 2026 17:05:55 +0200 Subject: [PATCH 014/391] docs(changes): write the 1.36.1 entries `CHANGES` carried one entry under 1.36.1 -- the route "if" documentation item -- against 49 commits since the 1.36.0 tag, roughly 27 of them user-visible. `docs/changes.xml` had the matching single-entry stub. Everything else merged this cycle was undocumented, including work a downstream reader has a direct interest in: the peer-supplied shared memory validation, the libunit bounds checks, the stale SCM_RIGHTS replay, the chunked request-body use-after-free, trailer parsing, the self-contradicting Content-Length rejection, and the removal of the TLS stack from the WebAssembly module. Seventeen entries added to both files, kept in step: one Feature, eleven Bugfix and five Change, ordered hardening and memory safety first, then the structural and performance changes. Each is written from the commit's own account of what went wrong and what an operator would observe, not from its subject line. The Feature entry covers the control API change from #205: validation errors now carry an RFC 6901 JSON Pointer in `location.path` and, on a close typo of a known member, a top-level `suggestion`. `CHANGES` is regenerated from `docs/changes.xml` with the wrapping the `docs/` XSLT produces, so the two files agree line for line for this section. Deliberately not listed: test-only commits, CI and workflow changes, Dependabot action bumps, and internal refactors with no observable effect. The four allocation-eliding changes (connection struct freelist, embedded static file context and buffer descriptor freelist, inline variable scratchpad, inline response fields) are folded into one entry, since separately they say nothing an operator can act on. The `1.36.1` date line still reads 02 Aug 2026, the date the stub was created. It needs setting to the actual release date at tag time; this commit does not guess it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014og6SESdCDxXzCpR6H9X33 --- CHANGES | 108 ++++++++++++++++++++++++++++++ docs/changes.xml | 169 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) diff --git a/CHANGES b/CHANGES index 69f92f593..c1c1ab4e6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,114 @@ Changes with FreeUnit 1.36.1 02 Aug 2026 + *) Feature: configuration validation errors now say where they are. A + rejected configuration request carries an RFC 6901 JSON Pointer in + the "path" member of "location": it points at the member whose value + failed validation, or at the containing object when the failure is an + unknown parameter, whose name the message itself already carries; "" + is the document root. On an unknown parameter whose name is a close + and unambiguous typo of a known one, the intended name is reported in + a top-level "suggestion" member. The addition is strictly additive: + the existing response fields and the error message wording are + unchanged, and each new member is omitted where it does not apply. + + *) Bugfix: validate the shared memory segment identifier and geometry an + application process supplies in its NXT_PORT_MSG_MMAP message. The + router checked only the pid fields, then used the peer-authored + header id directly as an array index and mapped the descriptor at + whatever length fstat() reported; a compromised or faulty application + could drive an out-of-range write and an oversized mapping. + + *) Bugfix: bounds-check the router-supplied chunk identifier and size in + libunit before consuming them, so an out-of-range size can no longer + make every serialized-pointer check in the arrival-time validation + block pass and be read out of bounds in the application process. + + *) Bugfix: do not replay a stale SCM_RIGHTS control message in libunit + when the embedder's port_recv callback returns without receiving + anything. The out parameter carrying the control-data length was + stored unconditionally, so a descriptor from an earlier message could + be adopted a second time. + + *) Bugfix: check the shared memory buffer allocation when forwarding a + websocket frame. Exhausting the outgoing shared memory crashed the + router with a NULL dereference; every other caller already checked. + + *) Bugfix: keep the chunked request body buffer alive across framing + reads. A buffer drained without producing a data slice was handed to + its completion handler while the parser still referenced it, a + use-after-free reachable from an ordinary chunked request. + + *) Bugfix: parse the HTTP/1.1 trailer section (RFC 9112 7.1.2) instead + of erroring on the first byte after the terminal chunk. In the proxy + response relay this aborted an already-streaming response and + truncated the body at a racy point; chunked requests carrying a + trailer were rejected the same way. Trailer field names and values + are validated as they are parsed. + + *) Bugfix: reject an application response that disagrees with itself + about its own body length. A duplicate Content-Length was neither + detected nor parsed, so every such header was forwarded verbatim and + the body was framed by the advertised length instead of chunked. + + *) Bugfix: release the response buffers adopted from the port on the + error paths of the router's response handler, and drain them from the + request output chain when the failure happens after they were linked + into it. Both leaked the chain the handler had taken ownership of. + + *) Bugfix: keep the memory pool cursor aligned in nxt_mp_get() by + rounding the requested size up to the maximum alignment rather than + flooring it. The small allocation path is a bump allocator with no + per-allocation alignment step, so any request whose size was not a + multiple of that alignment left the page cursor misaligned, and + allocations from that page kept violating the documented alignment + guarantee until a later size happened to bring the cursor back into + step, which crashed with SIGBUS on strict alignment targets such as + 32-bit ARM. + + *) Bugfix: serialize the process reference count on the runtime process + mutex and take a reference during cross-thread process lookups. The + count was mutated non-atomically from every engine thread and a + lookup returned an unreferenced process, so a router worker could + operate on an nxt_process_t -- including its shared memory segments + and its mutex -- that the main thread had already torn down. + + *) Bugfix: keep the socket and timer tasks of a connection scoped to + that connection, so a work item cannot run against a task belonging + to a different connection. + + *) Change: remove the dead chunk-tracking bitmap from the shared memory + segment header. Its trailing fields ran past the header area and + aliased the payload of the segment's first chunk, which was harmless + only because nothing read or wrote them; the field offsets a peer of + either vintage uses are unchanged, so old and new builds still + interoperate. A static assertion now pins the header struct inside + the area reserved for it. + + *) Change: the shared memory chunk range check is now a single shared + implementation used by both the router and libunit, with the chunk + count computed so that a peer-supplied size in the top of the 32-bit + range cannot wrap to zero chunks and be accepted. + + *) Change: recycle connection structures through a per-engine freelist, + embed the static file context in the request, recycle static file + buffer descriptors through a thread-local freelist, and resolve the + first eight routing variables and the first sixteen response header + fields from arrays embedded in the request. Each avoids or reuses an + allocation on a per-connection or per-request hot path. + + *) Change: the WebAssembly component module no longer supports + guest-initiated outbound HTTP. Serving Unit never needed it, and the + feature that provided it linked an entire TLS stack -- rustls, + tokio-rustls, webpki-roots and rustls-webpki -- into the module. A + guest that only imports wasi:http/outgoing-handler, as the + wasi:http/proxy world requires, still instantiates and runs; an + actual outbound call now fails with HttpRequestDenied. + + *) Change: update wasmtime from 36.0.12 to 47.0.2 in the WebAssembly + component module, keeping outbound HTTP denied and the TLS stack out + of the dependency graph. + *) Change: the "if" option of a route "match" object, added in 1.33.0, is now described in the OpenAPI specification and covered by the test suite. A request matches the step only when the condition is true; diff --git a/docs/changes.xml b/docs/changes.xml index 3f0d2387b..c43fd4070 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,175 @@ date="2026-08-02" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +configuration validation errors now say where they are. A rejected +configuration request carries an RFC 6901 JSON Pointer in the "path" member +of "location": it points at the member whose value failed validation, or at +the containing object when the failure is an unknown parameter, whose name +the message itself already carries; "" is the document root. On an unknown +parameter whose name is a close and unambiguous typo of a known one, the +intended name is reported in a top-level "suggestion" member. The addition +is strictly additive: the existing response fields and the error message +wording are unchanged, and each new member is omitted where it does not +apply. + + + + + +validate the shared memory segment identifier and geometry an application +process supplies in its NXT_PORT_MSG_MMAP message. The router checked only +the pid fields, then used the peer-authored header id directly as an array +index and mapped the descriptor at whatever length fstat() reported; a +compromised or faulty application could drive an out-of-range write and an +oversized mapping. + + + + + +bounds-check the router-supplied chunk identifier and size in libunit before +consuming them, so an out-of-range size can no longer make every +serialized-pointer check in the arrival-time validation block pass and be +read out of bounds in the application process. + + + + + +do not replay a stale SCM_RIGHTS control message in libunit when the +embedder's port_recv callback returns without receiving anything. The out +parameter carrying the control-data length was stored unconditionally, so a +descriptor from an earlier message could be adopted a second time. + + + + + +check the shared memory buffer allocation when forwarding a websocket frame. +Exhausting the outgoing shared memory crashed the router with a NULL +dereference; every other caller already checked. + + + + + +keep the chunked request body buffer alive across framing reads. A buffer +drained without producing a data slice was handed to its completion handler +while the parser still referenced it, a use-after-free reachable from an +ordinary chunked request. + + + + + +parse the HTTP/1.1 trailer section (RFC 9112 7.1.2) instead of erroring on +the first byte after the terminal chunk. In the proxy response relay this +aborted an already-streaming response and truncated the body at a racy +point; chunked requests carrying a trailer were rejected the same way. +Trailer field names and values are validated as they are parsed. + + + + + +reject an application response that disagrees with itself about its own body +length. A duplicate Content-Length was neither detected nor parsed, so every +such header was forwarded verbatim and the body was framed by the advertised +length instead of chunked. + + + + + +release the response buffers adopted from the port on the error paths of the +router's response handler, and drain them from the request output chain when +the failure happens after they were linked into it. Both leaked the chain +the handler had taken ownership of. + + + + + +keep the memory pool cursor aligned in nxt_mp_get() by rounding the requested +size up to the maximum alignment rather than flooring it. The small +allocation path is a bump allocator with no per-allocation alignment step, so +any request whose size was not a multiple of that alignment left the page +cursor misaligned, and allocations from that page kept violating the +documented alignment guarantee until a later size happened to bring the +cursor back into step, which crashed with SIGBUS on strict alignment targets +such as 32-bit ARM. + + + + + +serialize the process reference count on the runtime process mutex and take a +reference during cross-thread process lookups. The count was mutated +non-atomically from every engine thread and a lookup returned an unreferenced +process, so a router worker could operate on an nxt_process_t -- including its +shared memory segments and its mutex -- that the main thread had already torn +down. + + + + + +keep the socket and timer tasks of a connection scoped to that connection, so +a work item cannot run against a task belonging to a different connection. + + + + + +remove the dead chunk-tracking bitmap from the shared memory segment header. +Its trailing fields ran past the header area and aliased the payload of the +segment's first chunk, which was harmless only because nothing read or wrote +them; the field offsets a peer of either vintage uses are unchanged, so old +and new builds still interoperate. A static assertion now pins the header +struct inside the area reserved for it. + + + + + +the shared memory chunk range check is now a single shared implementation +used by both the router and libunit, with the chunk count computed so that a +peer-supplied size in the top of the 32-bit range cannot wrap to zero chunks +and be accepted. + + + + + +recycle connection structures through a per-engine freelist, embed the static +file context in the request, recycle static file buffer descriptors through a +thread-local freelist, and resolve the first eight routing variables and the +first sixteen response header fields from arrays embedded in the request. +Each avoids or reuses an allocation on a per-connection or +per-request hot path. + + + + + +the WebAssembly component module no longer supports guest-initiated outbound +HTTP. Serving Unit never needed it, and the feature that provided it linked +an entire TLS stack -- rustls, tokio-rustls, webpki-roots and rustls-webpki -- +into the module. A guest that only imports wasi:http/outgoing-handler, as the +wasi:http/proxy world requires, still instantiates and runs; an actual +outbound call now fails with HttpRequestDenied. + + + + + +update wasmtime from 36.0.12 to 47.0.2 in the WebAssembly component module, +keeping outbound HTTP denied and the TLS stack out of the dependency graph. + + + the "if" option of a route "match" object, added in 1.33.0, is now From af749985a7639fc5cabc902c6edc20f9dcc74e35 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 20 Aug 2026 21:28:14 +0200 Subject: [PATCH 015/391] test(lifecycle): make the app fd check immune to lazy port discovery test_app_lifecycle_app_fds_stable compared two measurement windows and failed when the second grew. That is not what a leak is, and CI said so: the test failed on java-26, java-25, java-21 and python for exactly one descriptor, each time on a different runtime, which is the shape of a race rather than of a leak. The race is lazy router-engine port discovery. The router runs nxt_ncpu engine threads, and an application does not learn an engine's port until the first request routed through that engine -- nxt_unit_process_new_port() keeps that NEW_PORT socket. Which engines a burst touches is up to the kernel's accept balancing, so a first touch can fall in any window. Measured on php with six plain 40-request windows, the application's socket count ran 8, 8, 9, 9, 9, 9: one discovery, after the second window, then flat. Pinning the router to one engine removes the variable at its source, and was tried first, but `listen_threads` is a live reconfiguration: restoring the default at teardown leaves conftest's own _check_fds() looking at a router that has rebuilt its engines, and it reports that as a 21-descriptor leak in the router. Requiring growth in BOTH windows was the first answer and it is not enough: discovery is one-time per ENGINE, not per process, so with two engines still undiscovered after the first window, windows two and three can each add a socket and a healthy run grows in both. It also costs the test its sensitivity, since a leak that shows up in one window only -- one descriptor per incoming segment, the leak the oversized POST exists to expose -- would have to repeat to be believed. So descriptors are counted per class and each class is held to the rule that fits it. count_fds() returns sockets, incoming shared-memory segments and request spool files separately: * shm and spool must be flat, in every window. Both are closed by the same message that delivered them (nxt_unit_process_msg()'s `done:`), so any growth at all is a leak, and one window is enough to say so. * sockets are bounded rather than flat. Total growth across the two windows may not exceed the engine ceiling -- nxt_ncpu, which is CPU_COUNT() of the affinity mask (src/nxt_lib.c), and the test does not touch `listen_threads`. Lazy discovery can never exceed that, however the first touches fall across the windows, including a whole burst's worth of fresh engines on a host with more CPUs than a burst has requests; a leak of one socket per request adds two bursts and is caught on any host under 2 * BURST CPUs. What this gives up is a socket leak that stays under the engine ceiling, which is unavoidable while discovery is live and is the class the churn tests above cover through unit.log alerts. The test's own claim is narrowed to match: it covers a leak per request and per incoming segment. It cannot cover the one-time paths -- a leak of recv_msg->fd[0] in nxt_unit_process_new_port() lands in a single window and is indistinguishable from the discovery it now tolerates -- because the application is loaded with processes=1 and never restarted or reconfigured here. Port-announcement churn is covered by the other tests in this module, through the alerts a bad close writes into unit.log. count_fds() is narrowed while here, for the same reason -- it counted things that are not unit's and can appear at any time. Shared memory counts only names carrying `unit.` (php's opcache maps /memfd:opcache_lock, and a JVM heap can appear as /memfd:java_heap). Spool files are matched by name rather than by "regular file under the temp dir": that directory is not unit's alone -- ApplicationJava.prepare_env() stages the whole webapp into {temp_dir}/java, so the directory rule counted the JVM's descriptors on WEB-INF/classes/*.class -- while a spool file is created as mkstemp("/req-XXXXXXXX") and unlinked at once (nxt_h1p_request_body_read()), so its link target is `{temp_dir}/req-XXXXXXXX (deleted)`. Matching that basename drops unit.log without needing a special case for it. fd_targets() labels its output by the same classes, so a failure message describes what was counted, and it returns lists rather than sets, so it cannot show fewer entries than the count that failed. The module's claim to cover request spool files is dropped: no test here reaches that path, because every body is either well under body_buffer_size or accompanied by a buffer deliberately raised above it. The count_fds() class stays as a guard, documented as one. A worker that disappears mid-measurement is now reported where it happens, inside the settle loop, rather than through a check on the collected counts that the loop's own fallback made unreachable. --- test/test_app_lifecycle.py | 325 ++++++++++++++++++++++++++++--------- 1 file changed, 251 insertions(+), 74 deletions(-) diff --git a/test/test_app_lifecycle.py b/test/test_app_lifecycle.py index 313b613c8..4944179de 100644 --- a/test/test_app_lifecycle.py +++ b/test/test_app_lifecycle.py @@ -45,6 +45,7 @@ import os import subprocess import time +from collections import namedtuple from pathlib import Path import pytest @@ -108,6 +109,12 @@ def load(self, script, **kwargs): # nxt_unit_response_buf_alloc() refuses anything above PORT_MMAP_DATA_SIZE, so # a request that fills a segment leaves it unable to build a response at all. # That is a property of the sample app, not of the port handling under test. +# +# The legacy `wasm` runtime is absent deliberately: the suite has no client +# for it (only test/unit/applications/lang/wasm_component.py exists), its +# build-test leg does not install or run pytest at all +# (`if: module != 'wasm'`), and sanitize.yml has no wasm leg. A row here +# would skip in every leg that exists. RUNTIMES = { 'libunit': (ApplicationLibunit, 'libunit', [], [], False, False), 'go': (ApplicationGo, 'mirror', ['go'], [], True, True), @@ -131,13 +138,19 @@ def load(self, script, **kwargs): BODY = '0123456789' * 200 -# PORT_MMAP_DATA_SIZE (src/nxt_port_memory_int.h): the payload area of one -# shared-memory segment, and therefore the amount of request data the router -# can hand an application before it has to create another one. +# PORT_MMAP_DATA_SIZE on a non-NXT_MMAP_TINY_CHUNK build +# (src/nxt_port_memory_int.h): the payload area of one shared-memory segment, +# and therefore the amount of request data the router can hand an application +# before it has to create another one. SEGMENT = 10 * 1024 * 1024 KEEPALIVE = {'Host': 'localhost', 'Connection': 'keep-alive'} +# Requests per measurement window in test_app_lifecycle_app_fds_stable(): the +# size of the burst is what turns "one descriptor per request" into a number +# no lazy allocation can reach. +BURST = 40 + class LifecycleApp: """Ties a runtime's client to its app name and request shape.""" @@ -166,9 +179,14 @@ def request(self, sock=None, keepalive=False, **kwargs): """One request that forces real request/response traffic. POST with a body where the runtime mirrors it back -- bodies travel - through the shared-memory and spool-file paths that carry descriptors - to the app -- plain GET otherwise. Returns (response, socket); the - socket is None unless the connection was kept alive. + through the shared-memory path that carries descriptors to the app -- + plain GET otherwise. Returns (response, socket); the socket is None + unless the connection was kept alive. + + BODY is small on purpose, and so no request here spools: a body past + body_buffer_size reaches the application as a descriptor to a file + instead, and the runtime that would suffer most from one is the wasm + component, whose read loop does not terminate on a short read. """ if keepalive or sock is not None: # A COPY. HTTP1.http() writes Content-Length into the dict it @@ -372,6 +390,9 @@ def app(request): return LifecycleApp(runtime, factory(), name, mirror, oversized) +FdCounts = namedtuple('FdCounts', ('sockets', 'shm', 'spool')) + + def count_fds(pid): """Descriptors of the kinds libunit hands an application process. @@ -383,10 +404,37 @@ def count_fds(pid): JVM, not in the port handling this test exists to watch. So count only what the docstring of the test actually claims: sockets - (ports), shared memory segments (incoming mmap), and regular files under - the test's own temp dir (request spool files). Anything the language - runtime opens for its own purposes is deliberately out of scope, and a - real leak of a port, a segment or a spool file still fails. + (ports) and shared memory segments (incoming mmap). Anything the + language runtime opens for its own purposes is deliberately out of scope, + and a real leak of a port or a segment still fails. + + Counted per class rather than as one total, because the classes do not + behave alike: an application acquires port sockets lazily, one per router + engine, so their count legitimately rises for a while, while a segment or + a spool descriptor is closed by the same message that delivered it and + must never rise at all. A single total would have to carry the socket + tolerance, and would then be blind to a per-segment leak. Separate + counts let the caller hold each class to its own rule. + + Segments are narrowed further, because "shared memory" is not + unit-specific -- php's opcache maps /memfd:opcache_lock, and a JVM heap + can appear as /memfd:java_heap -- so only names carrying `unit.` count. + + Spool files are narrowed by name, not by "regular file under the temp + dir". The temp dir is not unit's alone: it holds unit.log, which every + worker keeps open for its whole life, and ApplicationJava.prepare_env() + stages the whole webapp into {temp_dir}/java + (test/unit/applications/lang/java.py), so a directory rule counts the + JVM's classloader descriptors on WEB-INF/classes/*.class -- exactly the + runtime-specific noise the narrowing above exists to drop. A spool file + is created as mkstemp("/req-XXXXXXXX") and unlinked immediately + (nxt_h1p_request_body_read(), src/nxt_h1proto.c), so its link target is + `{temp_dir}/req-XXXXXXXX (deleted)`; match that basename. + + The class is kept as a guard, but no test here reaches it: bodies are + either 2000 bytes (well under the 16 KiB default body_buffer_size) or + sent by test_app_lifecycle_app_fds_stable, which raises that buffer above + the body precisely so the body does NOT spool. """ fds = Path(f'/proc/{pid}/fd') @@ -394,7 +442,9 @@ def count_fds(pid): if not fds.is_dir(): return None - n = 0 + sockets = 0 + shm = 0 + spool = 0 try: for fd in fds.iterdir(): @@ -405,47 +455,89 @@ def count_fds(pid): # the descriptor closed underneath us; it is not a leak continue - if ( - target.startswith('socket:') - or target.startswith('/memfd:') - or target.startswith('/dev/shm/') - or target.startswith(f'{option.temp_dir}/') + if target.startswith('socket:'): + sockets += 1 + + elif target.startswith('/memfd:') or target.startswith( + '/dev/shm/' ): - n += 1 + if 'unit.' in target: + shm += 1 + + elif target.startswith(f'{option.temp_dir}/'): + if _is_spool(target): + spool += 1 except OSError: return None - return n + return FdCounts(sockets, shm, spool) + + +def _is_spool(target): + """Is this /proc//fd link target a request spool file?""" + + deleted = ' (deleted)' + + if target.endswith(deleted): + target = target[: -len(deleted)] + + return os.path.basename(target).startswith('req-') def fd_targets(pid): - """What the counted descriptors of `pid` point at, for a failure message. + """What the descriptors of `pid` point at, for a failure message. A bare count says growth happened; this says what grew, which is the difference between a report someone can act on and one that starts an investigation from nothing. + + A list rather than a set: two descriptors on the same target are exactly + what a leak looks like, and deduplicating them would print fewer entries + than the count that failed. + + Labelled by the classes count_fds() counts, so the report and the number + that failed describe the same thing; everything else goes to `other`, + which is printed but never counted -- when growth is real it is useful to + see what the runtime is holding, and when it is not, its absence from the + counted classes is the answer. """ + targets = {'sockets': [], 'shm': [], 'spool': [], 'other': []} + fds = Path(f'/proc/{pid}/fd') if not fds.is_dir(): - return set() - - targets = set() + return targets try: for fd in fds.iterdir(): try: - targets.add(os.readlink(fd)) + target = os.readlink(fd) except OSError: continue + if target.startswith('socket:'): + targets['sockets'].append(target) + + elif ( + target.startswith('/memfd:') or target.startswith('/dev/shm/') + ) and 'unit.' in target: + targets['shm'].append(target) + + elif target.startswith(f'{option.temp_dir}/') and _is_spool( + target + ): + targets['spool'].append(target) + + else: + targets['other'].append(target) + except OSError: - return set() + return targets - return targets + return {name: sorted(items) for name, items in targets.items()} def shm_segments(pid): @@ -673,36 +765,100 @@ def test_app_lifecycle_app_fds_stable(app): """No per-request descriptor growth inside the application process. conftest's _check_fds() only watches main/router/controller. The libunit - paths that hand descriptors to an app -- incoming mmap segments, request - spool files, port sockets -- are only observable here, so a leak of one - descriptor per request or per port announcement is invisible to the rest - of the suite. - - Measured across TWO identical bursts rather than before-and-after one. - A single burst cannot tell a leak from lazy one-time allocation, and the - app process does allocate lazily: a further incoming mmap segment or an - additional router port arrives when it is first needed, not during the - warm-up. Counting one window made this fail for a single descriptor on - java-26 and again on python, both times for growth that never repeated. - A leak is growth that keeps happening, so the claim is about the SECOND - window: whatever the first burst settles, the second must not add to it. + paths that hand descriptors to an app -- incoming mmap segments and port + sockets -- are only observable here, so a leak of one descriptor per + request or per incoming segment is invisible to the rest of the suite. + + Per request and per incoming segment is the whole claim. A leak in the + one-time paths -- recv_msg->fd[0] in nxt_unit_process_new_port(), say -- + is NOT covered: this test loads the application with processes=1 and + never restarts or reconfigures it, so every NEW_PORT it sees is a first + discovery, lands in a single window, and is indistinguishable from the + legitimate lazy acquisition described below. Port-announcement churn is + covered by the other tests in this module, and only through the alerts a + bad close writes into unit.log (Log.check_alerts()). + + Request spool files are the third kind of descriptor an application is + handed, and this module deliberately does not exercise them: every body + it sends is either well under body_buffer_size or accompanied by a + buffer raised above it. count_fds() still counts them, as a guard for + whoever adds the coverage. + + Measured across THREE identical bursts rather than before-and-after one, + and per descriptor class rather than as one total. A single burst cannot + tell a leak from lazy one-time allocation, and the app process does + allocate lazily. Counting one window made this fail for a single + descriptor on java-26 and again on python, both times for growth that + never repeated. + + Only sockets allocate lazily, so only sockets pay for it: + + * shm and spool descriptors must be flat. Both are closed by the same + message that delivered them -- nxt_unit_process_msg()'s `done:` -- + so any growth at all, in either window, is a leak. Every burst + forces an incoming segment (below), which keeps a per-segment leak + visible in a single window rather than cancelling out. + + * sockets are bounded, not flat. The router runs nxt_ncpu engine + threads (rtcf->threads, src/nxt_router.c), an application does not + learn an engine's port until the first request routed through that + engine, and nxt_unit_process_new_port() keeps that NEW_PORT socket + (src/nxt_unit.c). Discovery is one-time PER ENGINE, not per + process: with several engines still undiscovered after the first + window, windows two and three can each add one, so "growth in both + windows" is a legitimate outcome and cannot be the rule. + + The socket rule is therefore one bound, and the discovery ceiling is + what makes it safe. There are at most nxt_ncpu engines, the test does + not touch `listen_threads`, and nxt_ncpu is CPU_COUNT() of the affinity + mask, capped by _SC_NPROCESSORS_ONLN (nxt_lib.c) -- which is what + os.sched_getaffinity() reports here. So: total socket growth across the + two windows may not exceed that ceiling. Discovery can never exceed it, + however the first touches fall across the windows -- one window, both, + or a whole burst's worth of fresh engines on a machine with more CPUs + than the burst has requests -- so this bound cannot be tripped by the + engine count. A per-window bound was tried and dropped for exactly that + last case: on a host with BURST or more undiscovered engines, BURST fresh + connections can legitimately retain BURST new port sockets in one window. + + What the socket rule gives up is a socket leak that stays under the + ceiling: one socket per request adds 2 * BURST over the two windows and + is caught on any machine with fewer than 2 * BURST CPUs -- every CI + runner by a wide margin -- but on a larger host, or for a leak slower + than one per request, growth that sums to no more than nxt_ncpu reads as + discovery. That is unavoidable while discovery is live, and it is the + class where a leak is cheapest to spot elsewhere -- a port descriptor + mishandled in the churn tests above shows up as an alert. The classes + with no tolerance at all are the ones only this test watches. + + Measured on php with this workload reduced to six plain 40-request + windows, the app's socket count ran 8, 8, 9, 9, 9, 9: one discovery, + after the second window, then flat. The bound passes that; a + per-request leak (8, 48, 88) trips it on any host under 80 CPUs. + + Pinning the router to one engine would remove the variable at its source, + and is the more obvious fix -- but `listen_threads` is a live + reconfiguration, and restoring the default at teardown leaves conftest's + own _check_fds() looking at a router that has rebuilt its engines, which + it reports as a 21-descriptor leak in the router. Each burst ends with one oversized POST, and that is not decoration. The router creates a segment only when the ones it has for this application cannot serve a request, and 2000-byte requests release their chunks as they complete, so a burst of them reuses the segment the warm-up caused - and never makes the router announce another. A descriptor leaked per - incoming segment would then already be inside BOTH counts and cancel out - -- the one leak this test names in its own docstring, invisible to it. - The oversized request forces an announcement; the burst asserts that it - happened, so the probe cannot quietly stop working. + and never makes the router announce another. A window that received no + segment cannot show a descriptor leaked per segment -- the one leak this + test names in its own docstring would be invisible to it, and the flat + rule would pass a leaking build. The oversized request forces an + announcement; the burst asserts that it happened, so the probe cannot + quietly stop working. The POST grows by one segment per burst because the pool grows with it: - the segment the first burst forced is free again by the time the second - runs, and would swallow a repeat of the same body. What stays equal - between the two windows is the thing being counted -- one segment - created, one descriptor received -- which is what makes the difference - between them a leak rather than a workload difference. + the segments an earlier burst forced are free again by the time the next + one runs, and would swallow a repeat of the same body. What stays equal + across the windows is the thing being counted -- one segment created, one + descriptor received -- which is what makes the differences between them + leaks rather than workload differences. Runtimes whose application cannot answer such a request keep the bursts and lose only the segment part of the claim (see RUNTIMES); driving one @@ -740,7 +896,7 @@ def test_app_lifecycle_app_fds_stable(app): app.request() def burst(pool): - for _ in range(40): + for _ in range(BURST): app.request() if app.oversized: @@ -754,48 +910,69 @@ def burst(pool): n = count_fds(pid) if n is None or n == settled: + settled = n break settled = n time.sleep(0.1) + # a disappearing worker is not a leak, but it is not a pass either: + # report it where it happened rather than as a count that is missing + assert settled is not None, ( + f'{app.runtime}: worker vanished mid-measurement' + ) + return settled if count_fds(pid) is None: pytest.skip('no /proc//fd on this platform') - warm = shm_segments(pid) - - first = burst(1) - after_first = shm_segments(pid) + counts = [] + segments = [shm_segments(pid)] - second = burst(2) - after_second = shm_segments(pid) + for pool in (1, 2, 3): + counts.append(burst(pool)) + segments.append(shm_segments(pid)) assert app.pids() == {pid}, f'{app.runtime}: same worker throughout' if app.oversized: - # Both windows, not just the second: a first burst that forced - # nothing would leave its own leak out of `first` and make the - # comparison below read a per-segment leak as one-time growth. - assert len(after_first) > len(warm), ( - f'{app.runtime}: no incoming segment in the first burst, so a ' - f'leak of one cannot be seen: {sorted(warm)} -> ' - f'{sorted(after_first)}' + # Every window, not just the measured pair: a window that forced no + # segment would leave its own leak out of the baseline and make a + # per-segment leak read as growth that stopped. + for i in range(3): + assert len(segments[i + 1]) > len(segments[i]), ( + f'{app.runtime}: no incoming segment in burst {i + 1}, so a ' + f'leak of one cannot be seen: {sorted(segments[i])} -> ' + f'{sorted(segments[i + 1])}' + ) + + # the classes libunit closes as it goes: flat, in every window + for name in ('shm', 'spool'): + series = [getattr(n, name) for n in counts] + growth = [series[i + 1] - series[i] for i in range(2)] + + assert max(growth) <= option.fds_threshold, ( + f'{app.runtime}: {name} descriptor leak in application process: ' + f'{series} across three repeated identical bursts; open now: ' + f'{fd_targets(pid)}' ) - assert len(after_second) > len(after_first), ( - f'{app.runtime}: no incoming segment in the second burst, so a ' - f'leak of one cannot be seen: {sorted(after_first)} -> ' - f'{sorted(after_second)}' - ) + # sockets: bounded by the engine count, because port discovery is lazy + # and one-time per engine. nxt_ncpu is CPU_COUNT() of the affinity mask + # (src/nxt_lib.c) and the test does not touch `listen_threads`. + ncpu = max(len(os.sched_getaffinity(0)), 1) - assert first is not None and second is not None, ( - f'{app.runtime}: worker vanished mid-measurement' - ) + sockets = [n.sockets for n in counts] + growth = [sockets[i + 1] - sockets[i] for i in range(2)] + + # only the rises are summed: a window that lost a socket must not buy + # the next one room to leak in + total = sum(n for n in growth if n > 0) - assert second - first <= option.fds_threshold, ( - f'{app.runtime}: descriptor leak in application process: ' - f'{first} -> {second} across a repeated identical burst; ' - f'open now: {sorted(fd_targets(pid))}' + assert total <= ncpu + option.fds_threshold, ( + f'{app.runtime}: socket leak in application process: {sockets} ' + f'across three repeated identical bursts, growing by {total} in ' + f'total where lazy port discovery can account for at most {ncpu} ' + f'(one per router engine); open now: {fd_targets(pid)}' ) From 62a9ded69738b1035fd5f9e34771e57eb8a065a6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 25 Aug 2026 15:20:59 +0200 Subject: [PATCH 016/391] otel: tie a request's span to the memory pool that holds it A span is allocated Rust-side at header parse time and handed to C as a raw Box. The only thing that ever gave it back was nxt_otel_span_collect(), reached solely through NXT_OTEL_COLLECT_STATE in nxt_otel_test_and_call_state(). A request that dies before its response headers never gets there: both nxt_http_request_init_state and nxt_http_request_body_state wire .error_handler straight to nxt_http_request_close_handler(), which releases r->mem_pool without an otel hook of any kind. So every client that opened a connection, sent a request line and walked away leaked one span, permanently, for as long as telemetry stayed enabled -- unbounded growth driven entirely by remote input. The obvious repair is a call to the otel code from the close/error handler, and it is the wrong one: it fixes the two exit paths that exist today and silently breaks again the first time someone adds a third. The span does not need another caller, it needs an owner. Registering an nxt_mp_cleanup() on r->mem_pool at the moment the span is created gives it one: the pool is released on every exit path by construction, so the span is released on every exit path by construction, including paths not yet written. nxt_http_static.c uses the same idiom to tie a file descriptor's lifetime to a request. The cleanup is a no-op for a request that finished normally, because nxt_otel_span_collect() NULLs r->otel->trace after handing the span over and is the only other place that does; that NULL is the whole of the double-release guard, so it has to stay true of any future path that ends a span. An abandoned request's span is ended and exported with Status::Error rather than dropped. An aborted or timed-out request is usually the trace someone is actually looking for, and a span that silently vanishes is indistinguishable from telemetry being broken; flagging it as failed also matches how 5xx responses are already recorded. Ending a span costs nothing here -- it only enqueues onto the Rust batch processor, which exports later from its own thread -- so the engine thread cannot stall inside pool teardown. The task captured at registration belongs to the connection, which may already have been recycled by the time the pool is destroyed, so the cleanup logs against the current thread's task instead. Verified against a running unitd with telemetry configured: a POST that announces Content-Length: 100, sends five bytes and closes now logs "otel: releasing the span of an unfinished request", while a request that completes normally does not. Co-Authored-By: Claude Opus 5 (1M context) --- src/nxt_otel.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/nxt_otel.c b/src/nxt_otel.c index 2d2eb1883..b2dce7a11 100644 --- a/src/nxt_otel.c +++ b/src/nxt_otel.c @@ -354,6 +354,63 @@ nxt_otel_span_collect(nxt_task_t *task, nxt_http_request_t *r) } +/* + * Release the span of a request that is being torn down. + * + * Registered as an r->mem_pool cleanup at span creation, so it runs from + * nxt_mp_destroy() on *every* request exit: the normal completion path, the + * client-abort/error path that goes straight to + * nxt_http_request_close_handler() without ever reaching COLLECT, and any exit + * path added in the future. Tying the span's lifetime to the pool that holds + * it makes the release structural rather than something each new exit path has + * to remember. + * + * nxt_otel_span_collect() NULLs r->otel->trace after handing the span to Rust, + * and it is the only other place that does so; that NULL is what keeps this + * idempotent, i.e. a no-op for a request that was collected normally. + */ +static void +nxt_otel_span_pool_cleanup(nxt_task_t *task, void *obj, void *data) +{ + nxt_thread_t *thr; + nxt_otel_state_t *state; + + state = obj; + + if (state->trace == NULL) { + return; + } + + thr = nxt_thread(); + + /* + * The task recorded at registration belongs to the connection, which may + * already have been recycled by the time the pool is destroyed, so log + * against the current thread's task instead of the one passed in. + */ + nxt_log(thr->task, NXT_LOG_DEBUG, + "otel: releasing the span of an unfinished request"); + + /* + * The request never produced a response. Mark the span failed and export + * it rather than dropping it silently: an aborted or timed out request is + * usually the interesting trace, and a span that simply vanishes is + * indistinguishable from telemetry being broken. Exporting also keeps the + * span consistent with the 5xx case, which is already flagged + * Status::Error in nxt_otel_span_add_status(). + * + * This cannot stall the engine thread: nxt_otel_rs_send_trace() only ends + * the span, which enqueues it on the Rust-side batch processor; the actual + * OTLP export runs later on that processor's own thread. + */ + nxt_otel_rs_set_error(state->trace); + nxt_otel_rs_send_trace(state->trace); + + state->trace = NULL; + state->status = NXT_OTEL_UNINIT_STATE; +} + + static void nxt_otel_error(nxt_task_t *task, nxt_http_request_t *r) { @@ -428,6 +485,27 @@ nxt_otel_trace_and_span_init(nxt_task_t *task, nxt_http_request_t *r) return; } + /* + * Bind the span to the request memory pool right away, so that from here + * on no exit path can drop the request without releasing the span. + */ + if (nxt_slow_path(nxt_mp_cleanup(r->mem_pool, nxt_otel_span_pool_cleanup, + task, r->otel, NULL) != NXT_OK)) + { + /* + * Without the cleanup nothing guarantees the span is released on an + * abort, so end it now instead of tracing this request and risking a + * leak. + */ + nxt_log(task, NXT_LOG_ERR, "couldn't register otel span cleanup"); + + nxt_otel_rs_send_trace(r->otel->trace); + r->otel->trace = NULL; + + nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); + return; + } + nxt_otel_state_transition(r->otel, NXT_OTEL_HEADER_STATE); } From 81d7c6630dc879008c10e0f8dd9439c37dee4aac Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 25 Aug 2026 15:21:28 +0200 Subject: [PATCH 017/391] otel: flush the span pipeline before the router exits nxt_otel_rs_uninit() had exactly two callers, both in the reconfigure path in nxt_router_conf_create(). Nothing called it when the process went away, so everything the batch processor was holding at that moment -- up to MAX_QUEUE_SIZE, 4096 spans, plus whatever was accumulating in the batch being assembled -- was discarded. A restart or a reload therefore threw away precisely the spans covering the window an operator is most likely to be looking at. The hook goes in nxt_runtime_exit(). It is the single funnel: the quit port message, the SIGTERM and SIGQUIT handlers and the internal failure paths all converge on nxt_runtime_quit() and from there on nxt_runtime_exit(), which is the last thing to run before exit(). Guarding on rt->type keeps it to the router, the only process that builds spans. The flush had to be bounded, or the fix would trade "loses spans on exit" for "takes ten seconds to stop", which is the more visible regression of the two: provider.shutdown() waits for the exporter's own EXPORT_TIMEOUT, and against an unreachable collector that is the full ten seconds sitting in front of exit(), felt by every supervisor and every restart. So the Rust side gains nxt_otel_rs_shutdown_bounded(), which takes the provider and the tokio runtime out of their static slots, *moves* them into a helper thread and waits up to a caller-supplied budget for that thread to finish. Taking them out first means the statics are left empty and unlocked no matter how the wait ends; on timeout we simply return and let the helper die with the process. Two seconds is the budget. A reachable collector needs a small fraction of that -- a local OTLP endpoint answers in milliseconds -- and a dead one is not worth more. The result is best-effort by construction: spans that cannot be flushed inside the budget are lost exactly as they are lost today, and a line at [warn] says so rather than leaving it to be guessed at. Verified both ways against a running unitd. With a batch_size larger than the number of pending spans, so that nothing would have been exported on its own, a local collector receives the OTLP POST only after SIGTERM, and shutdown takes 0.12s. Pointed at a blackholed address, shutdown takes 2.03s and logs the warning, where an unbounded flush would have taken ten seconds. Co-Authored-By: Claude Opus 5 (1M context) --- src/nxt_otel.c | 57 +++++++++++++++ src/nxt_otel.h | 18 +++++ src/nxt_runtime.c | 13 ++++ src/otel/src/lib.rs | 164 +++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 250 insertions(+), 2 deletions(-) diff --git a/src/nxt_otel.c b/src/nxt_otel.c index b2dce7a11..f6e278bcc 100644 --- a/src/nxt_otel.c +++ b/src/nxt_otel.c @@ -726,6 +726,63 @@ nxt_otel_parse_tracestate(void *ctx, nxt_http_field_t *field, uintptr_t data) } +/* + * Flush the telemetry pipeline on the way out of the process. + * + * Called from nxt_runtime_exit(), the single funnel every router exit passes + * through (signal handlers, the "quit" port message and internal failures all + * converge on nxt_runtime_quit() -> nxt_runtime_exit() -> exit()). Without + * this the batch processor's queue -- up to MAX_QUEUE_SIZE (4096) spans plus + * whatever is in the batch being assembled -- is simply discarded. + * + * The flush is bounded rather than unconditional. A blocking shutdown waits + * for the exporter's own 10s timeout when the collector is unreachable, which + * would trade "loses spans on exit" for "takes ten seconds to stop", a far + * more visible regression for anything supervising unitd. The budget below is + * comfortably more than a reachable collector needs (a local OTLP endpoint + * answers in milliseconds) and short enough that a dead one is not felt. + */ + +#define NXT_OTEL_EXIT_FLUSH_TIMEOUT_MS 2000 + +void +nxt_otel_shutdown(nxt_task_t *task) +{ + if (!nxt_otel_rs_is_init()) { + return; + } + + nxt_log(task, NXT_LOG_DEBUG, "otel: flushing spans before exit"); + + /* + * NXT_OTEL_SHUTDOWN_FLUSHED is not a promise that nothing was lost: the + * router's worker engines are still live here, so a span ended after the + * flush reaches an already shut down provider and is dropped silently. + * That one stays unreportable until producers can be quiesced, which is + * the P5 graceful-shutdown work, tracked in issue #219. + */ + switch (nxt_otel_rs_shutdown_bounded(NXT_OTEL_EXIT_FLUSH_TIMEOUT_MS)) { + + case NXT_OTEL_SHUTDOWN_TIMEOUT: + nxt_log(task, NXT_LOG_WARN, + "otel: the final span flush did not complete within %d ms, " + "exiting anyway; the spans it held are lost", + NXT_OTEL_EXIT_FLUSH_TIMEOUT_MS); + break; + + case NXT_OTEL_SHUTDOWN_FAILED: + nxt_log(task, NXT_LOG_WARN, + "otel: span export to the collector failed, exiting anyway; " + "the spans of at least one batch are lost"); + break; + + default: + nxt_log(task, NXT_LOG_DEBUG, "otel: span flush complete"); + break; + } +} + + void nxt_otel_log_callback(nxt_uint_t log_level, const char *arg) { diff --git a/src/nxt_otel.h b/src/nxt_otel.h index 3e64983d1..38d32398c 100644 --- a/src/nxt_otel.h +++ b/src/nxt_otel.h @@ -32,6 +32,23 @@ extern void nxt_otel_rs_add_attr(void *trace, nxt_str_t *key, nxt_str_t *val); extern void nxt_otel_rs_set_error(void *trace); extern uint8_t nxt_otel_rs_is_init(void); extern void nxt_otel_rs_uninit(void); +extern uint8_t nxt_otel_rs_shutdown_bounded(uint64_t timeout_ms); + +/* + * Return values of nxt_otel_rs_shutdown_bounded(). Defined on the Rust side + * as NXT_OTEL_SHUTDOWN_* in src/otel/src/lib.rs; the two lists must be kept + * in step. + * + * TIMEOUT the flush did not answer within the caller's budget. + * FLUSHED the flush completed and no export has failed since the provider + * was installed. Not a promise that no spans were lost: see the + * producer race documented on the Rust side and in issue #219. + * FAILED an export failed -- either the one this call forced, or an earlier + * batch recorded by the sticky failure flag. + */ +#define NXT_OTEL_SHUTDOWN_TIMEOUT 0 +#define NXT_OTEL_SHUTDOWN_FLUSHED 1 +#define NXT_OTEL_SHUTDOWN_FAILED 2 #endif @@ -78,6 +95,7 @@ void nxt_otel_log_callback(nxt_uint_t log_level, const char *arg); void nxt_otel_test_and_call_state(nxt_task_t *task, nxt_http_request_t *r); void nxt_otel_request_error_path(nxt_task_t *task, nxt_http_request_t *r); +void nxt_otel_shutdown(nxt_task_t *task); #endif /* _NXT_OTEL_H_INCLUDED_ */ diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index abc9a0fe3..4e1aa2f63 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -11,6 +11,9 @@ #include #include #include +#if (NXT_HAVE_OTEL) +#include +#endif static nxt_int_t nxt_runtime_inherited_listen_sockets(nxt_task_t *task, @@ -614,6 +617,16 @@ nxt_runtime_exit(nxt_task_t *task, void *obj, void *data) return; } +#if (NXT_HAVE_OTEL) + /* + * Only the router builds spans, and this is the last point before exit() + * at which the batch processor can still be flushed. + */ + if (rt->type == NXT_PROCESS_ROUTER) { + nxt_otel_shutdown(task); + } +#endif + if (rt->type == NXT_PROCESS_MAIN) { if (rt->pid_file != NULL) { nxt_file_delete(rt->pid_file); diff --git a/src/otel/src/lib.rs b/src/otel/src/lib.rs index 9311491dd..a5e885e3d 100644 --- a/src/otel/src/lib.rs +++ b/src/otel/src/lib.rs @@ -21,12 +21,16 @@ use opentelemetry::trace::{ }; use opentelemetry::{Context, KeyValue}; use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig}; +use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::trace::{ - BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider, + BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider, SpanData, + SpanExporter as SdkSpanExporter, }; use opentelemetry_sdk::Resource; use std::ffi::{c_char, CStr, CString}; use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; use std::sync::Mutex; use std::time::Duration; use std::{ptr, slice}; @@ -40,6 +44,14 @@ const SPAN_NAME: &str = "request"; const NXT_LOG_ERR: nxt_uint_t = 1; +/// Return values of `nxt_otel_rs_shutdown_bounded`. +/// +/// These are mirrored as `NXT_OTEL_SHUTDOWN_*` in `src/nxt_otel.h`, which is +/// the only consumer; the two lists must be kept in step. +const NXT_OTEL_SHUTDOWN_TIMEOUT: u8 = 0; +const NXT_OTEL_SHUTDOWN_FLUSHED: u8 = 1; +const NXT_OTEL_SHUTDOWN_FAILED: u8 = 2; + #[repr(C)] pub struct nxt_str_t { pub length: usize, @@ -69,6 +81,52 @@ fn runtime_slot() -> &'static Mutex> { &RT } +/// Set whenever an export returns `Err`, and cleared when a new provider is +/// installed. Sticky because the `BatchSpanProcessor` throws away the result +/// of every batch it exports on its own schedule (it only propagates the one +/// forced by `force_flush`), so without this a batch that failed minutes +/// before exit would leave nothing at all for the shutdown path to report. +static EXPORT_FAILED: AtomicBool = AtomicBool::new(false); + +/// Wraps the configured OTLP exporter so a failed export is remembered in +/// `EXPORT_FAILED` even when the processor discards the result. +/// +/// Everything other than `export` is delegated verbatim to the inner +/// exporter: taking the trait's defaults here would silently turn shutdown, +/// force-flush and resource propagation into no-ops. +#[derive(Debug)] +struct FailureTrackingExporter { + inner: E, +} + +impl SdkSpanExporter for FailureTrackingExporter { + async fn export(&self, batch: Vec) -> OTelSdkResult { + let res = self.inner.export(batch).await; + + if res.is_err() { + EXPORT_FAILED.store(true, Ordering::Release); + } + + res + } + + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.inner.shutdown_with_timeout(timeout) + } + + fn shutdown(&self) -> OTelSdkResult { + self.inner.shutdown() + } + + fn force_flush(&self) -> OTelSdkResult { + self.inner.force_flush() + } + + fn set_resource(&mut self, resource: &Resource) { + self.inner.set_resource(resource); + } +} + /// Build the OTLP/HTTP exporter: the blocking reqwest client, no async runtime. fn build_http_exporter(endpoint: String) -> Result { SpanExporter::builder() @@ -187,7 +245,17 @@ pub unsafe extern "C" fn nxt_otel_rs_init( } }; - let processor = BatchSpanProcessor::builder(exporter) + // A new provider starts with a clean slate, so the shutdown path does not + // normally report a failure that belonged to the exporter we replaced: + // nxt_otel_rs_shutdown_tracer() above joins the old worker thread before + // this runs. It joins on a 5s budget though, and an export gets 10s, so + // a worker still blocked on a dead collector can outlive the join and set + // the flag after this store. The cost is one spurious FAILED at the next + // exit, describing a failure that was real but belonged to the old + // exporter -- not worth a second flag to suppress. + EXPORT_FAILED.store(false, Ordering::Release); + + let processor = BatchSpanProcessor::builder(FailureTrackingExporter { inner: exporter }) .with_batch_config( BatchConfigBuilder::default() .with_max_export_batch_size(batch_size as usize) @@ -356,6 +424,98 @@ pub unsafe extern "C" fn nxt_otel_rs_send_trace(trace: *mut BoxedSpan) { span.end(); } +/// Flush and tear down the live tracer provider with a caller-supplied bound, +/// for use on the process exit path. +/// +/// `nxt_otel_rs_shutdown_tracer` can block for as long as the exporter's own +/// `EXPORT_TIMEOUT` (10s) when the collector is unreachable, which is far too +/// long to sit in front of `exit()`. The provider and runtime are taken out of +/// their slots here and *moved* into a helper thread, so the statics are left +/// empty and unlocked whatever happens; we then wait up to `timeout_ms` for +/// that thread to finish flushing. On timeout we give up and return: the +/// helper thread is left running and dies with the process. Best effort by +/// construction — spans that could not be flushed in the budget are lost, +/// exactly as they are today, but a dead collector can no longer delay exit. +/// +/// Returns one of three statuses, mirrored in `src/nxt_otel.h`: +/// +/// * `NXT_OTEL_SHUTDOWN_FLUSHED` (1) — the flush completed within the budget +/// and no export has failed since this provider was installed. +/// * `NXT_OTEL_SHUTDOWN_FAILED` (2) — an export failed. Either the flush this +/// call forced failed, or `EXPORT_FAILED` records an earlier batch that did: +/// the processor exports on its own schedule and discards the result, so the +/// only way that failure is visible at all is the sticky flag. The helper +/// thread failing to spawn reports here too; it is a failure, not a timeout. +/// * `NXT_OTEL_SHUTDOWN_TIMEOUT` (0) — the helper thread did not answer inside +/// `timeout_ms`. Kept distinct from the above because a rejected export +/// fails in milliseconds, and calling that a timeout sends an operator +/// looking for latency that is not there. +/// +/// One loss remains unreportable from here: the router's worker engines are +/// still live when `nxt_runtime_exit` calls this, so a span they end after the +/// flush reaches a provider that is already shut down and is dropped without a +/// word. Nothing at this layer can see it — the engine threads are never +/// joined (`nxt_thread_join` has no call sites), so producers cannot be +/// quiesced first. That needs the P5 graceful-shutdown work and is tracked in +/// issue #219. A `FLUSHED` return therefore means "this flush succeeded and no +/// export has failed", not "no spans were lost". +#[no_mangle] +pub unsafe extern "C" fn nxt_otel_rs_shutdown_bounded(timeout_ms: u64) -> u8 { + let provider = provider_slot().lock().ok().and_then(|mut g| g.take()); + let rt = runtime_slot().lock().ok().and_then(|mut g| g.take()); + + if provider.is_none() && rt.is_none() { + return if EXPORT_FAILED.load(Ordering::Acquire) { + NXT_OTEL_SHUTDOWN_FAILED + } else { + NXT_OTEL_SHUTDOWN_FLUSHED + }; + } + + let (tx, rx) = mpsc::channel::(); + + let spawned = std::thread::Builder::new() + .name("otel-shutdown".to_string()) + .spawn(move || { + let mut flushed = true; + + if let Some(provider) = provider { + // Needs the runtime alive on a grpc build, hence the ordering. + // + // force_flush() first, because it is the only one of the two + // that reports whether the spans actually left: the batch + // processor hands shutdown() the export result and shutdown() + // throws it away, reporting only that the worker answered. A + // collector that refuses the connection fails in milliseconds, + // so shutdown() alone would call that a clean flush and exit + // without a word. Both run whatever the first one says. + let exported = provider.force_flush().is_ok(); + let stopped = provider.shutdown().is_ok(); + + flushed = exported && stopped; + } + if let Some(rt) = rt { + rt.shutdown_background(); + } + // Read the sticky flag only after the flush above, so a batch the + // flush itself pushed out and failed on is counted here too. + let _ = tx.send(flushed && !EXPORT_FAILED.load(Ordering::Acquire)); + }); + + // The caller logs these, so they must not be conflated: a collector that + // rejects the export fails in milliseconds, and reporting that as a + // timeout sends an operator looking for latency that is not there. + match spawned { + Ok(_) => match rx.recv_timeout(Duration::from_millis(timeout_ms)) { + Ok(true) => NXT_OTEL_SHUTDOWN_FLUSHED, + Ok(false) => NXT_OTEL_SHUTDOWN_FAILED, + Err(_) => NXT_OTEL_SHUTDOWN_TIMEOUT, + }, + // Nothing flushed at all, and no time was spent trying. + Err(_) => NXT_OTEL_SHUTDOWN_FAILED, + } +} + /// Flush and tear down the live tracer provider, if any. #[no_mangle] pub unsafe extern "C" fn nxt_otel_rs_shutdown_tracer() { From 0603409ad83fe8500d2146af927c6df7e43a9625 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 25 Aug 2026 15:21:48 +0200 Subject: [PATCH 018/391] router: rebuild the otel exporter only when telemetry changed nxt_router_conf_create() called nxt_otel_rs_init() on every config apply that had a /settings/telemetry section, whether or not that section had changed. nxt_otel_rs_init() flushes the live provider and builds a new one, so adding a route or touching an application -- work with no connection to telemetry at all -- blocked the router's control thread for as long as the pending batch took to export, up to the exporter's ten second timeout when the collector is unreachable, and dropped whatever was in flight across the swap. The control thread is the one answering the API, so the config request hangs with it. Caching what was last handed to the exporter makes an unchanged section a no-op. The endpoint and protocol strings point into the configuration memory pool, which is released when the apply completes, so they are copied. The comparison is deliberately narrow. Only exact equality counts as unchanged; anything it cannot establish falls through to the rebuild, which is the old behaviour, so the conservative direction is also the cheap one. The two doubles are compared exactly rather than with a tolerance -- they are re-parsed from the same JSON text on every apply, so an unchanged section reproduces them bit for bit, and the only cost of an unexpected inequality is one rebuild. nxt_otel_rs_is_init() is consulted as well, so that an init which failed inside the Rust side -- an exporter that could not be built -- is retried on the next apply instead of being remembered as successfully applied. Removing the telemetry section still tears down, and clears the cache with it. Verified against a running unitd pointed at a blackholed collector with spans queued: two successive route-only applies now complete in 11ms each, while changing the endpoint still pays the flush at 5s. Before this, every one of the three paid it. Co-Authored-By: Claude Opus 5 (1M context) --- src/nxt_router.c | 128 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 4 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 5ac3b2748..866d358c5 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1620,6 +1620,104 @@ static nxt_conf_map_t nxt_router_websocket_conf[] = { }; +#if (NXT_HAVE_OTEL) + +/* + * The telemetry settings currently applied to the Rust exporter. + * + * nxt_otel_rs_init() flushes the live provider and builds a new one, which + * blocks the router's control thread for as long as the export of the pending + * batch takes -- up to the exporter's 10s timeout against an unreachable + * collector -- and drops whatever is in flight across the swap. Every config + * apply used to pay that, including applies that do not touch + * /settings/telemetry at all (adding a route, changing an app). Caching what + * was applied lets an unchanged telemetry section be a no-op. + * + * Only exact equality is treated as unchanged; anything the comparison cannot + * establish falls through to the rebuild, which is the old behaviour. + */ +typedef struct { + nxt_bool_t configured; + nxt_str_t endpoint; + nxt_str_t protocol; + double sample_fraction; + double batch_size; +} nxt_otel_applied_conf_t; + + +static nxt_otel_applied_conf_t nxt_otel_applied_conf; + + +static nxt_bool_t +nxt_router_otel_conf_unchanged(nxt_str_t *endpoint, nxt_str_t *protocol, + double sample_fraction, double batch_size) +{ + /* + * The doubles are re-parsed from the same JSON text on every apply, so an + * unchanged section reproduces them bit for bit; an exact comparison is + * therefore the right test here, and any inequality only costs a rebuild. + */ + return nxt_otel_applied_conf.configured + && nxt_strstr_eq(&nxt_otel_applied_conf.endpoint, endpoint) + && nxt_strstr_eq(&nxt_otel_applied_conf.protocol, protocol) + && nxt_otel_applied_conf.sample_fraction == sample_fraction + && nxt_otel_applied_conf.batch_size == batch_size; +} + + +static nxt_int_t +nxt_router_otel_conf_store_str(nxt_str_t *dst, const nxt_str_t *src) +{ + u_char *p; + + p = NULL; + + if (src->length != 0) { + p = nxt_malloc(src->length); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + memcpy(p, src->start, src->length); + } + + nxt_free(dst->start); + + dst->start = p; + dst->length = src->length; + + return NXT_OK; +} + + +/* + * Remember what was just handed to nxt_otel_rs_init(). The strings point into + * the configuration memory pool, which is released once this apply completes, + * so they have to be copied. On any failure the cache is invalidated, which + * only means the next apply rebuilds unconditionally. + */ +static void +nxt_router_otel_conf_remember(nxt_str_t *endpoint, nxt_str_t *protocol, + double sample_fraction, double batch_size) +{ + if (nxt_slow_path(nxt_router_otel_conf_store_str( + &nxt_otel_applied_conf.endpoint, endpoint) != NXT_OK + || nxt_router_otel_conf_store_str( + &nxt_otel_applied_conf.protocol, + protocol) != NXT_OK)) + { + nxt_otel_applied_conf.configured = 0; + return; + } + + nxt_otel_applied_conf.sample_fraction = sample_fraction; + nxt_otel_applied_conf.batch_size = batch_size; + nxt_otel_applied_conf.configured = 1; +} + +#endif + + static nxt_int_t nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, u_char *start, u_char *end) @@ -2225,11 +2323,33 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, ? nxt_conf_get_number(otel_sampling) : NXT_OTEL_SAMPLING_DEFAULT; - nxt_otel_rs_init(&nxt_otel_log_callback, &telemetry_endpoint, - &telemetry_proto, telemetry_sample_fraction, - telemetry_batching); - } else { + /* + * Rebuild the exporter only when the telemetry settings actually + * differ. nxt_otel_rs_is_init() is required as well so that a + * previous init that failed inside the Rust side (an exporter that + * could not be built) is retried rather than remembered as applied. + */ + if (!nxt_router_otel_conf_unchanged(&telemetry_endpoint, + &telemetry_proto, + telemetry_sample_fraction, + telemetry_batching) + || !nxt_otel_rs_is_init()) + { + nxt_otel_rs_init(&nxt_otel_log_callback, &telemetry_endpoint, + &telemetry_proto, telemetry_sample_fraction, + telemetry_batching); + + nxt_router_otel_conf_remember(&telemetry_endpoint, &telemetry_proto, + telemetry_sample_fraction, + telemetry_batching); + } + + } else if (nxt_otel_applied_conf.configured || nxt_otel_rs_is_init()) { nxt_otel_rs_uninit(); + + nxt_free(nxt_otel_applied_conf.endpoint.start); + nxt_free(nxt_otel_applied_conf.protocol.start); + nxt_memzero(&nxt_otel_applied_conf, sizeof(nxt_otel_applied_conf)); } #endif From f175dca77c1e794b9d833ac1f50108b7939b41bb Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 25 Aug 2026 21:41:50 +0200 Subject: [PATCH 019/391] otel: count spans by export outcome The exporter wrapper added for the shutdown path knows only whether some export has ever failed. That boolean cannot separate one transient blip from a collector that has been refusing everything for hours, which is the objection issue #219 raises against it. Count spans instead: two atomics, incremented by the batch length on either side of the export result, reset with the sticky flag when a new provider is installed. Spans rather than batches because the question an operator has is "how much telemetry did I lose", and a batch is a variable-size unit that answers it only by accident. The boolean stays: it is exactly the fact the shutdown path asks for, it survives the degenerate empty batch that would move no counter, and deriving it from the counter would put #218's three shutdown statuses at risk for no gain. nxt_otel_rs_export_stats() reads the pair out for the next commit's /status support; it reports whether a provider is installed at all, so a caller can tell "no telemetry configured" from "configured, nothing exported yet". --- src/nxt_otel.h | 7 +++++ src/otel/src/lib.rs | 65 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/nxt_otel.h b/src/nxt_otel.h index 38d32398c..42c5f640c 100644 --- a/src/nxt_otel.h +++ b/src/nxt_otel.h @@ -33,6 +33,13 @@ extern void nxt_otel_rs_set_error(void *trace); extern uint8_t nxt_otel_rs_is_init(void); extern void nxt_otel_rs_uninit(void); extern uint8_t nxt_otel_rs_shutdown_bounded(uint64_t timeout_ms); +/* + * Span export health for the /status API: writes the number of spans the + * exporter accepted and the number it rejected since the live provider was + * installed, and returns non-zero when a provider is installed at all. When + * it returns 0 the two counters are meaningless and must not be reported. + */ +extern uint8_t nxt_otel_rs_export_stats(uint64_t *exported, uint64_t *failed); /* * Return values of nxt_otel_rs_shutdown_bounded(). Defined on the Rust side diff --git a/src/otel/src/lib.rs b/src/otel/src/lib.rs index a5e885e3d..c6c1be746 100644 --- a/src/otel/src/lib.rs +++ b/src/otel/src/lib.rs @@ -29,7 +29,7 @@ use opentelemetry_sdk::trace::{ use opentelemetry_sdk::Resource; use std::ffi::{c_char, CStr, CString}; use std::str::FromStr; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc; use std::sync::Mutex; use std::time::Duration; @@ -88,6 +88,19 @@ fn runtime_slot() -> &'static Mutex> { /// before exit would leave nothing at all for the shutdown path to report. static EXPORT_FAILED: AtomicBool = AtomicBool::new(false); +/// Spans in batches the exporter accepted, and spans in batches it rejected, +/// since the live provider was installed. Read out by +/// `nxt_otel_rs_export_stats` for the `/status` API, which is the only way an +/// operator can see export health without waiting for the process to exit. +/// +/// Counted in spans rather than batches because the number an operator needs +/// is "how much telemetry did I lose", and a batch is a variable-size unit +/// that answers that only by accident. Neither counter includes spans the +/// processor dropped before an export was attempted (a full queue): that +/// happens inside the SDK, which reports it nowhere we can observe. +static SPANS_EXPORTED: AtomicU64 = AtomicU64::new(0); +static SPANS_FAILED: AtomicU64 = AtomicU64::new(0); + /// Wraps the configured OTLP exporter so a failed export is remembered in /// `EXPORT_FAILED` even when the processor discards the result. /// @@ -101,10 +114,20 @@ struct FailureTrackingExporter { impl SdkSpanExporter for FailureTrackingExporter { async fn export(&self, batch: Vec) -> OTelSdkResult { + // Count before the batch is moved into the inner exporter. + let spans = batch.len() as u64; + let res = self.inner.export(batch).await; if res.is_err() { + // The boolean is kept alongside the counter rather than derived + // from it: it is the fact the shutdown path asks for ("has any + // export failed"), and it stays true even for the degenerate + // empty batch that would add nothing to SPANS_FAILED. EXPORT_FAILED.store(true, Ordering::Release); + SPANS_FAILED.fetch_add(spans, Ordering::Relaxed); + } else { + SPANS_EXPORTED.fetch_add(spans, Ordering::Relaxed); } res @@ -196,6 +219,44 @@ pub unsafe extern "C" fn nxt_otel_rs_is_init() -> u8 { .unwrap_or(0) } +/// Report span export health for the `/status` API. +/// +/// Writes the number of spans the exporter accepted and the number it +/// rejected since the live provider was installed, and returns 1 when a +/// provider is installed at all. +/// +/// The counters are written whether or not a provider is installed, so on a +/// 0 return they hold whatever the previous provider left behind -- they are +/// only zeroed when a new one is installed. A 0 return therefore means the +/// values are stale, not absent: the caller must ignore them rather than +/// report them. +/// +/// The counters are read with two independent `Relaxed` loads, so a report +/// taken while an export is completing can be off by one batch. That is +/// deliberate: this is a health gauge, not an accounting ledger, and taking a +/// lock on the export path to make the pair atomic would cost more than the +/// skew is worth. +/// +/// # Safety +/// +/// `exported` and `failed` must each be either null or a valid, aligned, +/// writable `uint64_t`. +#[no_mangle] +pub unsafe extern "C" fn nxt_otel_rs_export_stats( + exported: *mut u64, + failed: *mut u64, +) -> u8 { + if !exported.is_null() { + *exported = SPANS_EXPORTED.load(Ordering::Relaxed); + } + + if !failed.is_null() { + *failed = SPANS_FAILED.load(Ordering::Relaxed); + } + + nxt_otel_rs_is_init() +} + #[no_mangle] pub unsafe extern "C" fn nxt_otel_rs_uninit() { nxt_otel_rs_shutdown_tracer(); @@ -254,6 +315,8 @@ pub unsafe extern "C" fn nxt_otel_rs_init( // exit, describing a failure that was real but belonged to the old // exporter -- not worth a second flag to suppress. EXPORT_FAILED.store(false, Ordering::Release); + SPANS_EXPORTED.store(0, Ordering::Relaxed); + SPANS_FAILED.store(0, Ordering::Relaxed); let processor = BatchSpanProcessor::builder(FailureTrackingExporter { inner: exporter }) .with_batch_config( From b802377f87a1fe1dc35fcf9ac71df3c00e00da79 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 25 Aug 2026 21:41:50 +0200 Subject: [PATCH 020/391] status: report otel span export health Export failures were observable only in the log line the router prints on its way out. A pipeline that silently stopped reaching the collector at 09:00 stayed invisible until the process was stopped, which is the half of issue #219 that #218 did not address. The router fills the span counters into the status report -- it is the process the exporter runs in, so it is the only place they can be read -- and the controller renders them as "telemetry": {"spans": {"exported": N, "failed": M}} Both halves are needed: a failure count alone cannot tell a healthy pipeline from one that has never exported anything. The object is omitted entirely, rather than zeroed, when OTel is not built in or "settings/telemetry" is absent, so /status is byte for byte what it was before on every configuration that does not use telemetry. The report fields are not built-gated, so the router-to-controller message keeps one layout regardless of how unit was configured. --- src/nxt_router.c | 13 +++++++++++++ src/nxt_status.c | 42 +++++++++++++++++++++++++++++++++++++++++- src/nxt_status.h | 13 +++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 866d358c5..122cf572f 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1001,6 +1001,19 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } nxt_queue_loop; +#if (NXT_HAVE_OTEL) + /* + * Span export health. The counters live in the Rust exporter wrapper, + * which runs in this process, so the router is the only place they can be + * read from. A build without OTel, or a configuration with no + * "settings/telemetry", leaves the memzero'd zeros in place and + * nxt_status_get() then omits the "telemetry" object entirely. + */ + report->otel_configured = nxt_otel_rs_export_stats( + &report->otel_spans_exported, + &report->otel_spans_failed); +#endif + report->apps_count = 0; app_stat = report->apps; p = b->mem.end; diff --git a/src/nxt_status.c b/src/nxt_status.c index 92cbf2e65..e3ecb9201 100644 --- a/src/nxt_status.c +++ b/src/nxt_status.c @@ -35,11 +35,21 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) static const nxt_str_t reqs_str = nxt_string("requests"); static const nxt_str_t total_str = nxt_string("total"); static const nxt_str_t apps_str = nxt_string("applications"); + static const nxt_str_t telemetry_str = nxt_string("telemetry"); + static const nxt_str_t spans_str = nxt_string("spans"); + static const nxt_str_t exported_str = nxt_string("exported"); + static const nxt_str_t failed_str = nxt_string("failed"); static const nxt_str_t procs_str = nxt_string("processes"); static const nxt_str_t run_str = nxt_string("running"); static const nxt_str_t start_str = nxt_string("starting"); - status = nxt_conf_create_object(mp, 4); + /* + * modules, connections, requests, applications -- plus "telemetry" when + * OTel is built in and currently configured. It is omitted rather than + * zeroed otherwise, so a build or a configuration without telemetry + * reports exactly what it did before. + */ + status = nxt_conf_create_object(mp, 4 + (report->otel_configured != 0)); if (nxt_slow_path(status == NULL)) { return NULL; } @@ -148,6 +158,36 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) nxt_conf_set_member_integer(obj, &total_str, report->requests, 0); + if (report->otel_configured) { + nxt_conf_value_t *spans; + + /* + * "is my telemetry reaching the collector?" needs both halves: a + * failure count alone cannot tell a healthy pipeline from one that + * has never exported anything. Both are cumulative since the live + * tracer provider was installed, so a telemetry reconfiguration + * restarts them. + */ + obj = nxt_conf_create_object(mp, 1); + if (nxt_slow_path(obj == NULL)) { + return NULL; + } + + nxt_conf_set_member(status, &telemetry_str, obj, idx++); + + spans = nxt_conf_create_object(mp, 2); + if (nxt_slow_path(spans == NULL)) { + return NULL; + } + + nxt_conf_set_member(obj, &spans_str, spans, 0); + + nxt_conf_set_member_integer(spans, &exported_str, + report->otel_spans_exported, 0); + nxt_conf_set_member_integer(spans, &failed_str, + report->otel_spans_failed, 1); + } + apps = nxt_conf_create_object(mp, report->apps_count); if (nxt_slow_path(apps == NULL)) { return NULL; diff --git a/src/nxt_status.h b/src/nxt_status.h index a99ac7d0e..8c370cba1 100644 --- a/src/nxt_status.h +++ b/src/nxt_status.h @@ -22,6 +22,19 @@ typedef struct { uint64_t closed_conns; uint64_t requests; + /* + * OpenTelemetry span export health, filled in by the router. The two + * counters are only meaningful when otel_configured is set: a build + * without OTel support, or one where "settings/telemetry" is absent, + * leaves them zero and reports no "telemetry" object in /status at all. + * + * Counted in spans since the live tracer provider was installed, i.e. + * reset by a telemetry reconfiguration. + */ + uint64_t otel_spans_exported; + uint64_t otel_spans_failed; + uint8_t otel_configured; + size_t apps_count; nxt_status_app_t apps[]; } nxt_status_report_t; From 260772b38c84656d9f5e1f095e356783d353f6b5 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 25 Aug 2026 21:41:50 +0200 Subject: [PATCH 021/391] test(otel): cover the /status span export counters Three cases: no "telemetry" object when telemetry is not configured (which is also the contract for a build without --otel, hence no probe), a live fake_otlp collector moving "exported" on both transports, and an endpoint with nothing listening moving "failed" -- the first automated check of the export-failure path at all, which until now was verified only by construction. FAKE_OTLP_BIN becomes overridable so the suite can run against a locally built collector; installing into /usr/local/bin needs root, which a developer machine may not offer. --- test/test_otel.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/test/test_otel.py b/test/test_otel.py index 3667e89bd..f7b607f39 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -17,7 +17,9 @@ # and OTLP/gRPC, so the transport under test is chosen by config (--protocol), # never by how the mock was built. CI installs it via the "Build fake_otlp" step # in ci.yml, mirroring fake_upstream. Skip gracefully when it is not built. -FAKE_OTLP_BIN = '/usr/local/bin/fake_otlp' +# Overridable so the suite can be run against a locally built fake_otlp +# without root (installing into /usr/local/bin needs it). +FAKE_OTLP_BIN = os.environ.get('FAKE_OTLP_BIN', '/usr/local/bin/fake_otlp') _skipif_no_fake_otlp = pytest.mark.skipif( not os.path.exists(FAKE_OTLP_BIN), @@ -763,3 +765,73 @@ def test_otel_sampling_ratio_nonfinite_rejected(): conf = client.conf(body) assert 'error' in conf, f'non-finite sampling_ratio must be rejected: {conf}' + + +def _status_telemetry(): + """The /status "telemetry" object, or None when it is absent.""" + return client.conf_get('/status').get('telemetry') + + +def _wait_for_spans(field, timeout=EXPORT_TIMEOUT, delay=0.1): + """Poll /status until telemetry/spans/ is non-zero; return the object.""" + deadline = time.time() + timeout + telemetry = _status_telemetry() + while time.time() < deadline: + if telemetry is not None and telemetry['spans'][field] > 0: + return telemetry + time.sleep(delay) + telemetry = _status_telemetry() + return telemetry + + +def test_otel_status_absent_without_telemetry(): + """/status carries no "telemetry" object when telemetry is not configured. + + True of a build without --otel as well, which is why this one does not + probe for OTel support first: the absence is the contract in both cases. + """ + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 200}}], + "applications": {}, + } + ) + + assert _status_telemetry() is None + + +@_skipif_no_fake_otlp +@pytest.mark.parametrize('protocol', ['http', 'grpc']) +def test_otel_status_counts_exported_spans(protocol): + """A span that reaches the collector is counted in /status (issue #219).""" + port = _get_free_port() + proc = _run_fake_otlp(port, protocol=protocol) + try: + _configure_or_skip(port, protocol=protocol) + + assert _get_until_header('traceparent')['status'] == 200 + + telemetry = _wait_for_spans('exported') + assert telemetry is not None, '/status must report telemetry when configured' + assert telemetry['spans']['exported'] > 0, 'exported spans must be counted' + assert telemetry['spans']['failed'] == 0, 'a live collector must not fail' + finally: + _kill(proc) + + +def test_otel_status_counts_failed_spans(): + """An export the collector refuses is counted as failed, while the process + is still running -- the whole point of #219: before this the only signal + was a log line at exit. + """ + # Nothing is listening here: the export is refused, not merely slow. + port = _get_free_port() + _configure_or_skip(port) + + assert _get_until_header('traceparent')['status'] == 200 + + telemetry = _wait_for_spans('failed') + assert telemetry is not None, '/status must report telemetry when configured' + assert telemetry['spans']['failed'] > 0, 'a refused export must be counted' + assert telemetry['spans']['exported'] == 0, 'nothing can have been exported' From c44f43ffa5e07b4bad555079ef15f6ae987a5759 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 22 Aug 2026 02:01:11 +0200 Subject: [PATCH 022/391] router: close the engine port when a worker thread exits Every router worker engine owns a port (engine->port): a socketpair -- port->pair[0] and port->pair[1] -- plus the memfd backing its message queue and that queue's mapping. Two references keep the port alive: the engine's own, and the one nxt_router_rt_add_port() takes when it publishes the port in rt->ports so the other threads can reach it. nxt_router_thread_exit_handler() (src/nxt_router.c) dropped only the first. It joins the dead worker thread, adopts the port's memory pool and calls nxt_port_use(task, port, -1); the rt->ports reference stayed, so the count never reached zero. And even had it reached zero, nxt_port_release() (src/nxt_port.c) only unlinks the port from its process and releases the pool -- descriptors are closed by nxt_port_close(), which nothing on this path called. The engine's epoll and eventfd are freed correctly by nxt_event_engine_free() just below, which is why the loss looked partial rather than absent. So every engine destroyed by a "settings/listen_threads" decrease leaked three descriptors -- pair[0], pair[1], queue_fd -- plus the queue mapping, for the lifetime of the router process, and no later increase gave them back. The regression test added alongside this measures a listen_threads 4 -> 1 -> 4 cycle: without the fix the router ends nine descriptors up, exactly three per destroyed engine. Close the port and unpublish it before dropping the reference, in that order, mirroring nxt_process_close_ports() (src/nxt_process.c). The handler runs on the main router engine -- the same thread that runs nxt_router_rt_add_port() -- so reaching into rt->ports here needs no lock, and nxt_runtime_port_remove() clears rt->port_by_type only when that slot holds this very port, which it never does for an engine port since nxt_router_rt_add_port() never sets it. nxt_thread_wait() above has already joined the worker, so nothing reads pair[0] any more. With both references gone the count reaches zero and nxt_port_release() runs, whose nxt_port_mp_cleanup() assertions that pair[0] and pair[1] are already -1 are now satisfied instead of being reached with live descriptors. The pre-existing "TODO notify all apps" above stays out of scope: what the applications hold for this engine is their own bookkeeping, and this change is only about what the router itself keeps open. --- src/nxt_router.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/nxt_router.c b/src/nxt_router.c index 122cf572f..8b5d4de8a 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -4378,6 +4378,18 @@ nxt_router_thread_exit_handler(nxt_task_t *task, void *obj, void *data) port->engine = task->thread->engine; nxt_mp_thread_adopt(port->mem_pool); + + /* + * The worker thread is gone, so nothing reads port->pair[0] any more: + * close the socket pair and the queue memfd, and drop the reference + * nxt_router_rt_add_port() took when it published the port in + * rt->ports. Without this every engine removed by a listen_threads + * decrease leaks 3 descriptors (2 socketpair + 1 memfd) for the + * router's lifetime. + */ + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + nxt_port_use(task, port, -1); nxt_mp_thread_adopt(engine->mem_pool); From 156ca659adf42f70fdd2ea0b7dc88a65e437e882 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 22 Aug 2026 02:01:28 +0200 Subject: [PATCH 023/391] test(router): pin descriptor accounting across listen_threads churn test_listen_threads.py drives the engine-teardown path the previous commit fixes: it pins "settings/listen_threads" to 4, shrinks to 1, grows back to 4, and requires the router's /proc descriptor count to come back to where it started. Serving a static share keeps it free of any language module, so it runs on every build. The high water mark is pinned rather than taken from the default (nxt_ncpu): the leak was per destroyed engine, so a cycle starting from the default would destroy nothing on a single-CPU machine and the test would pass everywhere for the wrong reason. Going 4 -> 1 -> 4 always destroys three engines, so the delta is the same on any host. Both the before reading and the after reading poll rather than sleep -- engines are created and destroyed asynchronously with respect to the configuration reply, so a reading taken right after it can catch the router mid-transition -- and the after loop returns its last reading whichever way it goes, so a leak fails as the count it is: AssertionError: router leaked 9 descriptors over a listen_threads 4 -> 1 -> 4 cycle (44 before, 53 after) The conftest teardown check would eventually have caught this too, but only as a generic "descriptors leak router", which is what several existing tests had to switch off. Three of them can now switch it back on, having only ever been suppressing this bug: - test_process_teardown_churn.py, whose comment described the leak in detail as a pre-existing defect; - test_process_shm_oosm.py, which pointed at that comment; - test_conn_recycle.py, which blamed the per-engine epoll, signalfd and eventfd instead -- those are freed correctly, and the check polls, so their churn settles well inside its window. test_process_abrupt_teardown.py keeps its skip: killing workers leaves descriptor accounting unsettled in all three long-lived processes while the respawn runs, which is a separate reason and still holds. Only the listen_threads clause is dropped from its comment. Verified locally against a --debug build with the php module: the four touched tests plus the new one pass with every descriptor check armed, and reverting src/nxt_router.c to its parent makes the new test fail on the assertion quoted above, with the teardown check reporting 21 -- the nine from the cycle plus twelve from the initial shrink off this 8-CPU host's default. --- test/test_conn_recycle.py | 7 +- test/test_listen_threads.py | 105 +++++++++++++++++++++++++++ test/test_process_abrupt_teardown.py | 6 +- test/test_process_shm_oosm.py | 8 +- test/test_process_teardown_churn.py | 11 +-- 5 files changed, 110 insertions(+), 27 deletions(-) create mode 100644 test/test_listen_threads.py diff --git a/test/test_conn_recycle.py b/test/test_conn_recycle.py index 3a359bb8e..5ab9368ac 100644 --- a/test/test_conn_recycle.py +++ b/test/test_conn_recycle.py @@ -13,12 +13,7 @@ @pytest.fixture(autouse=True) -def setup_method_fixture(temp_dir, skip_fds_check): - # Thread-count changes intentionally add and remove each engine's epoll, - # signalfd, and eventfd descriptors, so the generic router FD baseline is - # not meaningful for this test. The recycler itself owns no descriptors. - skip_fds_check(router=True) - +def setup_method_fixture(temp_dir): Path(f'{temp_dir}/assets').mkdir(parents=True) Path(f'{temp_dir}/assets/index.html').write_text(BODY, encoding='utf-8') diff --git a/test/test_listen_threads.py b/test/test_listen_threads.py new file mode 100644 index 000000000..6c258524f --- /dev/null +++ b/test/test_listen_threads.py @@ -0,0 +1,105 @@ +import time +from pathlib import Path + +import pytest +from conftest import _count_fds, pid_by_name + +from unit.applications.proto import ApplicationProto + +client = ApplicationProto() + +BODY = '0123456789' + +# Enough engines that shrinking destroys several of them on any host. The +# leak is per destroyed engine, so a cycle that starts from the default +# (nxt_ncpu) would destroy nothing on a single-CPU machine; pinning the high +# water mark here makes the delta the same everywhere. +THREADS = 4 + + +@pytest.fixture(autouse=True) +def setup_method_fixture(temp_dir): + Path(f'{temp_dir}/assets').mkdir(parents=True) + Path(f'{temp_dir}/assets/index.html').write_text(BODY, encoding='utf-8') + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"share": f'{temp_dir}/assets$uri'}}], + } + ) + + yield + + # The no-restart suite preserves /settings between tests. + assert 'success' in client.conf_delete('settings/listen_threads') + + +def _router_fds(): + return _count_fds(pid_by_name('unit: router')) + + +def _settle_fds(stable=5, timeout=100): + # Engines are created and destroyed asynchronously with respect to the + # configuration reply, so a reading taken right after it can catch the + # router mid-transition. Wait for the count to hold still instead of + # sleeping a fixed amount. + fds = _router_fds() + same = 0 + + for _ in range(timeout): + time.sleep(0.1) + + cur = _router_fds() + + same = same + 1 if cur == fds else 0 + fds = cur + + if same >= stable: + break + + return fds + + +def _wait_fds(expected, timeout=100): + # Return the last reading either way, so a leak fails as the count it is + # rather than as an opaque timeout. + for _ in range(timeout): + fds = _router_fds() + + if fds <= expected: + break + + time.sleep(0.1) + + return fds + + +def _set_threads(threads): + assert 'success' in client.conf( + {"listen_threads": threads}, 'settings' + ), f'listen_threads {threads}' + + assert client.get()['status'] == 200, f'request with {threads} engines' + + +def test_listen_threads_engine_fds(): + # A destroyed router engine used to keep its port socketpair and queue + # memfd open for the router's lifetime (3 descriptors each), because + # nxt_router_thread_exit_handler() only dropped one of the port's two + # references and nxt_port_release() closes nothing. Every shrink of + # "listen_threads" therefore cost descriptors that no later grow + # returned. + _set_threads(THREADS) + + before = _settle_fds() + + _set_threads(1) + _set_threads(THREADS) + + after = _wait_fds(before) + + assert after <= before, ( + f'router leaked {after - before} descriptors over a listen_threads' + f' {THREADS} -> 1 -> {THREADS} cycle ({before} before, {after} after)' + ) diff --git a/test/test_process_abrupt_teardown.py b/test/test_process_abrupt_teardown.py index 3088148a3..c7f92f369 100644 --- a/test/test_process_abrupt_teardown.py +++ b/test/test_process_abrupt_teardown.py @@ -137,10 +137,8 @@ def _wait_for_pids_gone(pids, timeout=5): def test_process_abrupt_teardown(skip_alert, skip_fds_check): # Killing workers leaves descriptor accounting unsettled in all three - # long-lived processes for as long as the respawn takes, and changing - # "listen_threads" leaks the destroyed engines' port socketpairs on top of - # that (see test_process_teardown_churn.py). Neither is what this test is - # about. + # long-lived processes for as long as the respawn takes, which is not what + # this test is about. skip_fds_check(main=True, router=True, controller=True) # Expected, and in fact asserted on below: this is how the main process diff --git a/test/test_process_shm_oosm.py b/test/test_process_shm_oosm.py index 8c5930a4a..08314ef6e 100644 --- a/test/test_process_shm_oosm.py +++ b/test/test_process_shm_oosm.py @@ -90,13 +90,7 @@ def _body(resp, size): return resp[-size:] -def test_process_shm_oosm(skip_fds_check): - # Same pre-existing engine-teardown descriptor leak that - # test_process_teardown_churn.py documents: changing "listen_threads" - # destroys event engines, and each destroyed engine keeps its port - # socketpair open. Unrelated to what this test exercises. - skip_fds_check(router=True) - +def test_process_shm_oosm(): client.load(APP, limits={"shm": SHM_LIMIT}, processes=1) assert 'success' in client.conf( diff --git a/test/test_process_teardown_churn.py b/test/test_process_teardown_churn.py index 2af4fd65e..f4cfc9f9a 100644 --- a/test/test_process_teardown_churn.py +++ b/test/test_process_teardown_churn.py @@ -105,16 +105,7 @@ def _wait_for_pids(count, timeout=100): return pids -def test_process_teardown_churn(skip_fds_check): - # Setting "listen_threads" changes the number of router worker engines, - # and destroying an engine leaks its descriptors: measured on this tree, - # shrinking the count frees each dead engine's epoll and eventfd but keeps - # its port socketpair open (3 sockets per destroyed engine), so the router - # ends the test with more descriptors than it started with. That is a - # pre-existing defect in engine teardown, not something this workload - # causes, and it is unrelated to what the test is here to exercise. - skip_fds_check(router=True) - +def test_process_teardown_churn(): client.load(APP, processes={"spare": SPARE, "max": 4, "idle_timeout": 5}) assert 'success' in client.conf( From a78cc63b7a01bda846788b64d9929d9234c50d01 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 22 Aug 2026 02:12:20 +0200 Subject: [PATCH 024/391] unit: warn, not alert, when a retired engine port is already closed nxt_unit_sendmsg() raises an alert for any sendmsg() failure while the context is online, on the premise that an online peer is reachable. That premise no longer holds for every port: since the previous commit the router closes an engine's port pair when a "listen_threads" decrease retires that engine (nxt_router_thread_exit_handler()), and it still does not tell the applications that hold a copy of the port -- the pre-existing "TODO notify all apps" in that handler. So if a client goes away after an application has taken its request but before it answers, the router drops the request and can retire the engine, and the application's eventual response to the reply port fails -- with ECONNREFUSED on the SOCK_DGRAM pairs Linux uses (src/nxt_socketpair.c), with EPIPE or ECONNRESET on a stream pair. Before the previous commit that write succeeded silently into the leaked, unread socket; now it fails, and the failure would be logged as an alert for what is a normal consequence of reconfiguration and not an application fault. Pass the port down to nxt_unit_sendmsg() and treat ECONNREFUSED, EPIPE and ECONNRESET on a per-engine port -- one learned through NEW_PORT/GET_PORT, i.e. neither the main router port nor the shared port -- as the warning it is: the only way to get them is the peer closing its end, and the request they carried was already abandoned on the router side. The main router port, the shared port and the readiness descriptor keep the alert for every error: a broken one of those is a real problem, and for some messages this log line is the only visible signal (nxt_unit_mmap_release() ignores the result of nxt_unit_send_shm_ack()). Notifying applications of a retired engine port would make the two sides symmetric and is left as a follow-up. --- src/nxt_unit.c | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 92ed26e15..cae7ecd09 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -175,8 +175,8 @@ static int nxt_unit_get_port(nxt_unit_ctx_t *ctx, nxt_unit_port_id_t *port_id); static ssize_t nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, const void *buf, size_t buf_size, const nxt_send_oob_t *oob); -static ssize_t nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd, - const void *buf, size_t buf_size, const nxt_send_oob_t *oob); +static ssize_t nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + int fd, const void *buf, size_t buf_size, const nxt_send_oob_t *oob); static int nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf); nxt_inline void nxt_unit_rbuf_cpy(nxt_unit_read_buf_t *dst, @@ -1058,7 +1058,7 @@ nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, int queue_fd) nxt_socket_msg_oob_init(&oob, fds); - res = nxt_unit_sendmsg(ctx, ready_fd, &msg, sizeof(msg), &oob); + res = nxt_unit_sendmsg(ctx, NULL, ready_fd, &msg, sizeof(msg), &oob); if (res != sizeof(msg)) { return NXT_UNIT_ERROR; } @@ -6223,7 +6223,7 @@ nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, msg.type = _NXT_PORT_MSG_READ_QUEUE; if (lib->callbacks.port_send == NULL) { - ret = nxt_unit_sendmsg(ctx, port->out_fd, &msg, + ret = nxt_unit_sendmsg(ctx, port, port->out_fd, &msg, sizeof(nxt_port_msg_t), NULL); nxt_unit_debug(ctx, "port{%d,%d} send %d read_queue", @@ -6269,7 +6269,7 @@ nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, (int) ret); } else { - ret = nxt_unit_sendmsg(ctx, port->out_fd, buf, buf_size, oob); + ret = nxt_unit_sendmsg(ctx, port, port->out_fd, buf, buf_size, oob); nxt_unit_debug(ctx, "port{%d,%d} sendmsg %d", (int) port->id.pid, (int) port->id.id, @@ -6281,12 +6281,13 @@ nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, static ssize_t -nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd, +nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int fd, const void *buf, size_t buf_size, const nxt_send_oob_t *oob) { int err; ssize_t n; struct iovec iov[1]; + nxt_unit_impl_t *lib; nxt_unit_ctx_impl_t *ctx_impl; iov[0].iov_base = (void *) buf; @@ -6323,9 +6324,36 @@ nxt_unit_sendmsg(nxt_unit_ctx_t *ctx, int fd, * too and cannot be used to distinguish steady state from * shutdown. The unambiguous flag is ctx_impl->online. */ + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - if (ctx_impl->online) { + /* + * A router engine port whose peer has already closed is not an + * application fault even while online: the router closes an + * engine's port pair when a "listen_threads" decrease retires + * that engine (nxt_router_thread_exit_handler()), and it does + * not tell the applications that still hold a copy of the port, + * so a response to a request that engine had handed out fails: + * with ECONNREFUSED on the SOCK_DGRAM pairs used on Linux + * (src/nxt_socketpair.c), with EPIPE or ECONNRESET on a stream + * pair. The request was abandoned on the router side already; + * say so at warn level. + * + * That reasoning covers only the per-engine ports learned through + * NEW_PORT/GET_PORT. The main router port, the shared port and + * the readiness descriptor have no such lifecycle: a broken one + * of those is a real problem and keeps the alert, because for + * some of its messages this log line is the only visible signal + * (nxt_unit_mmap_release() ignores nxt_unit_send_shm_ack()'s + * return value). + */ + if (ctx_impl->online + && !(port != NULL + && port != lib->router_port + && port != lib->shared_port + && (err == ECONNREFUSED || err == EPIPE + || err == ECONNRESET))) + { nxt_unit_alert(ctx, "sendmsg(%d, %d) failed: %s (%d)", fd, (int) buf_size, strerror(err), err); From 7658053dea1e102ec0a70e89e91dcf331bda4361 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 21 Aug 2026 22:47:40 +0200 Subject: [PATCH 025/391] main: close both descriptors on the whoami path A port message can carry two descriptors whatever its type normally uses: nxt_socket_msg_oob_get() accepts an SCM_RIGHTS payload of one or two, and the port dispatcher does not reclaim what a handler leaves behind. The whoami handler consumes fd[0] and its fail label closes fd[0], so a peer that attaches a second descriptor to a forged WHOAMI leaks one descriptor of the main process per message. Close the message's descriptors through nxt_port_recv_msg_close_fds() instead. The success path hands fd[0] to the port and clears it before falling through, so the descriptor the port owns is untouched. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KMTKtFcf3YhmBPvTSKTkfg --- src/nxt_main_process.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index d42b3ef37..b6d6832a6 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -802,9 +802,12 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) fail: - if (msg->fd[0] != -1) { - nxt_fd_close(msg->fd[0]); - } + /* + * Close both descriptors: WHOAMI carries one, but a compromised sender + * can attach a second to any message, and leaving it open here would + * leak a descriptor of the main process on every forged message. + */ + nxt_port_recv_msg_close_fds(msg); } From eacaf14f2f0d17d2d193487e0258399affcccd13 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 21 Aug 2026 22:47:40 +0200 Subject: [PATCH 026/391] router: close both descriptors on the conf data path The handler returned without closing anything when the reply port was not found, leaking every descriptor attached to the message, and its cleanup path closed only fd[0]. Since a peer can attach a second descriptor to any message type, both paths leak descriptors of the router. Close them through nxt_port_recv_msg_close_fds(). fd[0] is already closed and cleared right after the configuration is mapped, so the call is a no-op for it on that path. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KMTKtFcf3YhmBPvTSKTkfg --- src/nxt_router.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 8b5d4de8a..aa3d322b9 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -776,6 +776,7 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) msg->port_msg.reply_port); if (nxt_slow_path(port == NULL)) { nxt_alert(task, "conf_data_handler: reply port not found"); + nxt_port_recv_msg_close_fds(msg); return; } @@ -848,10 +849,12 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_mem_munmap(p, size); } - if (msg->fd[0] != -1) { - nxt_fd_close(msg->fd[0]); - msg->fd[0] = -1; - } + /* + * Close both descriptors: the configuration arrives on fd[0], but a + * compromised sender can attach a second one to any message and would + * otherwise leak a descriptor of the router on every forged message. + */ + nxt_port_recv_msg_close_fds(msg); } From e8b7f6444f3dfe643f8af2343eb1a6c88f93b490 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 21 Aug 2026 22:47:54 +0200 Subject: [PATCH 027/391] port: handle a malformed PROCESS_READY without a trace nxt_port_process_ready_handler() guarded its assumptions with nxt_assert(), which compiles out in release builds: a repeated PROCESS_READY re-ran the queue mapping and dropped the previous one, and a message naming a process with no ports reached nxt_process_port_first() on an empty queue. Neither path closed the descriptors the message carried, and the unknown-pid path did not close them either, so a peer can exhaust the descriptor table of whichever process handles the message -- main or the prototype. Close the message's descriptors on every path that does not consume them, and turn the empty-ports assertion into a runtime check. That check now runs before the process is marked ready, which deviates from the order the assertions had: a rejected message must leave the process as it found it. Otherwise a forged message naming a process that is still starting would mark it ready, so the next -- legitimate -- PROCESS_READY would find nothing left to do, and nxt_main_process_sigchld_handler() would clear the start stream of a process that never finished starting, dropping the REMOVE_PID that cancels the pending start RPC. A repeated PROCESS_READY is deliberately not rejected. The message is not authenticated, so refusing it would let a forged one that arrives first make itself permanent and lock the legitimate sender out of its own queue -- a worse outcome than the leak, since before this commit the real message could still replace the forged mapping. It keeps replacing it, and the previous mapping and its descriptor are now released rather than leaked. Check the mapping for MAP_FAILED as the new port handler does, rather than storing it in port->queue for a later dereference: nxt_port_close() munmaps it on != NULL, and nxt_port_queue_send() would have used it. Transfer ownership of the descriptor explicitly, in this handler and in nxt_port_new_port_handler(), so that a message the port has taken from cannot be closed twice by a later close_fds() call. The lookup deliberately stays on the self-declared msg->port_msg.pid. The kernel-validated sender PID cannot replace it here: the prototype installs this handler too, and there SCM_CREDENTIALS carries the namespace-local pid of a pid-isolated worker while the runtime hash is keyed on the global one. Authenticating this handler needs to accept either pid of the process it resolves, which belongs with the sender-type ACL the TODO in this file already tracks. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KMTKtFcf3YhmBPvTSKTkfg --- auto/sources | 1 + src/nxt_port.c | 90 ++++++++- src/test/nxt_port_ready_test.c | 325 +++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 5 files changed, 414 insertions(+), 7 deletions(-) create mode 100644 src/test/nxt_port_ready_test.c diff --git a/auto/sources b/auto/sources index 117910888..aa75a7600 100644 --- a/auto/sources +++ b/auto/sources @@ -177,6 +177,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_conf_parse_test.c \ src/test/nxt_port_fail_test.c \ src/test/nxt_port_mmap_range_test.c \ + src/test/nxt_port_ready_test.c \ " diff --git a/src/nxt_port.c b/src/nxt_port.c index 0d2ffe018..8e398b1d7 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -313,6 +313,10 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) port->pair[0] = -1; port->pair[1] = msg->fd[0]; + + /* The port owns the descriptor now. */ + msg->fd[0] = -1; + port->max_size = new_port_msg->max_size; port->max_share = new_port_msg->max_share; @@ -327,6 +331,7 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) void nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { + void *mem; nxt_port_t *port; nxt_process_t *process; nxt_runtime_t *rt; @@ -335,28 +340,99 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* Unreferenced: main and prototype processes run a single engine. */ + /* + * The lookup stays on msg->port_msg.pid. The kernel-validated sender + * PID cannot replace it here: the prototype installs this handler too, + * and there SCM_CREDENTIALS carries the namespace-local pid of a + * pid-isolated worker, while the runtime hash is keyed on the global + * one (see nxt_proto_process_created_handler(), which uses the two as + * distinct values, and nxt_process_created() refusing to register an + * isolated pid). Authenticating this handler needs to accept either + * pid of the process it resolves; see the sender-ACL TODO above. + */ process = nxt_runtime_process_find(rt, msg->port_msg.pid); if (nxt_slow_path(process == NULL)) { + nxt_port_recv_msg_close_fds(msg); return; } - nxt_assert(process->state != NXT_PROCESS_STATE_READY); + /* + * A repeated PROCESS_READY is not rejected: the message is not + * authenticated, so refusing it would let a forged one arriving first + * make itself permanent and lock the legitimate sender out of its own + * queue. It replaces the mapping instead, as it did before this check + * existed, and the previous mapping is released below rather than + * leaked. nxt_assert() is not used because it compiles out in release + * builds, leaving no diagnostic at all. + */ + if (nxt_slow_path(process->state == NXT_PROCESS_STATE_READY)) { + nxt_log(task, NXT_LOG_WARN, "repeated PROCESS_READY claiming " + "process %PI", msg->port_msg.pid); + } - process->state = NXT_PROCESS_STATE_READY; + /* + * Both guards reject before the state is set: a message that is not + * acted upon must leave no trace. Marking a process ready and only + * then rejecting it would make the next, legitimate PROCESS_READY trip + * the guard above, and would let nxt_main_process_sigchld_handler() + * clear the start stream of a process that never finished starting, + * dropping the REMOVE_PID that cancels the pending start RPC. + */ + if (nxt_slow_path(nxt_queue_is_empty(&process->ports))) { + nxt_log(task, NXT_LOG_WARN, "PROCESS_READY claiming process %PI, " + "which has no ports", msg->port_msg.pid); + nxt_port_recv_msg_close_fds(msg); + return; + } - nxt_assert(!nxt_queue_is_empty(&process->ports)); + process->state = NXT_PROCESS_STATE_READY; port = nxt_process_port_first(process); nxt_debug(task, "process %PI ready", msg->port_msg.pid); if (msg->fd[0] != -1) { - port->queue_fd = msg->fd[0]; - port->queue = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), - PROT_READ | PROT_WRITE, MAP_SHARED, - msg->fd[0], 0); + mem = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), + PROT_READ | PROT_WRITE, MAP_SHARED, msg->fd[0], 0); + if (nxt_fast_path(mem != MAP_FAILED)) { + /* + * Release what a previous PROCESS_READY installed, so that + * replacing the queue does not leak the old mapping and its + * descriptor. The size follows nxt_port_close() -- an app + * queue is mapped for the port with the reserved id. + */ + if (port->queue_fd != -1) { + nxt_fd_close(port->queue_fd); + port->queue_fd = -1; + } + + if (port->queue != NULL) { + nxt_mem_munmap(port->queue, + (port->id == (nxt_port_id_t) -1) + ? sizeof(nxt_app_queue_t) + : sizeof(nxt_port_queue_t)); + port->queue = NULL; + } + + port->queue_fd = msg->fd[0]; + port->queue = mem; + + /* The port owns the descriptor now. */ + msg->fd[0] = -1; + + } else { + nxt_log(task, NXT_LOG_WARN, "process %PI ready: queue mmap " + "failed %E, falling back to the socket", msg->port_msg.pid, + nxt_errno); + } } + /* + * Close whatever the sender attached and the port did not take over: + * fd[1] is never used here, and fd[0] survives a failed mapping. + */ + nxt_port_recv_msg_close_fds(msg); + nxt_port_send_new_port(task, rt, port, msg->port_msg.stream); } diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c new file mode 100644 index 000000000..61241ee4f --- /dev/null +++ b/src/test/nxt_port_ready_test.c @@ -0,0 +1,325 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership in + * nxt_port_process_ready_handler() (src/nxt_port.c). + * + * PROCESS_READY normally carries the queue descriptor of the process that + * announces itself, and the handler hands it to that process's port. A + * message naming a pid the runtime does not know took an early return that + * closed nothing, and a repeated READY re-ran the mapping and dropped the + * previous one -- both leak a descriptor of the receiver, which is the + * main or the prototype process. The dispatcher does not reclaim fds once + * the handler returns (see nxt_port_recv_msg_close_fds()), so a peer that + * attaches descriptors to forged messages can exhaust a privileged + * process's descriptor table. + * + * A message can carry two descriptors regardless of how many its type + * normally uses, so the test attaches both and expects both back. + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + +#if (NXT_HAVE_MEMFD_CREATE) +#include +#include +#endif + + +static nxt_bool_t +nxt_port_ready_test_fd_is_open(nxt_fd_t fd) +{ + return fcntl(fd, F_GETFD) != -1; +} + + +static nxt_int_t +nxt_port_ready_test_case(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, const char *name) +{ + nxt_fd_t fd0, fd1; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (fd0 == -1 || fd1 == -1) { + nxt_log_alert(thr->log, "port ready test failed to open /dev/null"); + goto fail; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_port_process_ready_handler(task, msg); + + if (nxt_port_ready_test_fd_is_open(fd0) + || nxt_port_ready_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port ready test: %s leaked a descriptor", + name); + goto fail; /* closes only what is still open */ + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port ready test: %s left fds in the message", + name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + /* + * Close only what the handler left open: the descriptors it consumed + * are gone already, and closing them again could reach an unrelated + * descriptor that reused the number. + */ + if (fd0 != -1 && nxt_port_ready_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_port_ready_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + return NXT_ERROR; +} + + +#if (NXT_HAVE_MEMFD_CREATE) + +/* + * Hand the handler a mappable descriptor and check that the port takes it + * over: the queue is installed, whatever the port held before is released, + * and the second descriptor of the message is closed. + */ +static nxt_int_t +nxt_port_ready_test_queue(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_port_t *port, const char *name) +{ + void *prev_queue; + nxt_fd_t fd, spare, prev_fd; + + /* + * The syscall form, as nxt_unit_shm_open() and nxt_unit_port_recv_test.c + * use: auto/shmem probes SYS_memfd_create, so a libc without the wrapper + * still defines NXT_HAVE_MEMFD_CREATE and this file has to link. + */ + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + spare = open("/dev/null", O_RDONLY); + + if (fd == -1 || spare == -1) { + nxt_log_alert(thr->log, "port ready test failed to create the queue"); + goto fail; + } + + if (ftruncate(fd, sizeof(nxt_port_queue_t)) == -1) { + nxt_log_alert(thr->log, "port ready test failed to size the queue"); + goto fail; + } + + prev_fd = port->queue_fd; + prev_queue = port->queue; + + msg->fd[0] = fd; + msg->fd[1] = spare; + + nxt_port_process_ready_handler(task, msg); + + if (port->queue_fd != fd || port->queue == NULL + || port->queue == MAP_FAILED) + { + nxt_log_alert(thr->log, "port ready test: %s did not install the " + "queue", name); + return NXT_ERROR; + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port ready test: %s left fds in the message", + name); + return NXT_ERROR; + } + + if (nxt_port_ready_test_fd_is_open(spare)) { + nxt_log_alert(thr->log, "port ready test: %s leaked the spare " + "descriptor", name); + return NXT_ERROR; + } + + if (prev_queue != NULL && nxt_port_ready_test_fd_is_open(prev_fd)) { + nxt_log_alert(thr->log, "port ready test: %s did not release the " + "previous queue", name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + if (fd != -1) { + (void) close(fd); + } + + if (spare != -1) { + (void) close(spare); + } + + return NXT_ERROR; +} + +#endif + + +nxt_int_t +nxt_port_ready_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_task_t *task; + nxt_int_t ret; + nxt_port_t *port; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *process; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ready test started"); + + task = thr->task; + task->thread = thr; + + /* Defined before the first goto done: the cleanup there releases it. */ + port = NULL; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + thr->runtime = rt; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + /* + * A pid the runtime does not know: the lookup fails and the handler + * has to close what the message carried. + */ + msg.port_msg.pid = nxt_pid + 1; + + ret = nxt_port_ready_test_case(thr, task, &msg, "unknown pid"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A process that is ready already: the state check has to reject the + * message rather than map the descriptor a second time. The process + * carries no ports, so reaching the mapping at all would be a bug -- + * the empty-ports guard is the second reject path covered here. + */ + process = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(process == NULL)) { + ret = NXT_ERROR; + goto done; + } + + process->pid = nxt_pid + 2; + process->state = NXT_PROCESS_STATE_CREATING; + nxt_queue_init(&process->ports); + + nxt_runtime_process_add(task, process); + + msg.port_msg.pid = process->pid; + + ret = nxt_port_ready_test_case(thr, task, &msg, "ready with no ports"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * The guard has to reject before the state is set, so that a message + * the handler refuses leaves the process as it found it. + */ + if (nxt_slow_path(process->state == NXT_PROCESS_STATE_READY)) { + nxt_log_alert(thr->log, "port ready test: a rejected message marked " + "the process ready"); + ret = NXT_ERROR; + goto done; + } + +#if (NXT_HAVE_MEMFD_CREATE) + + /* + * With a port on the process the handler takes the descriptor over and + * maps it, and a second PROCESS_READY replaces that mapping instead of + * leaking it -- a repeated message is not refused, so that a forged one + * arriving first cannot lock the legitimate sender out. + */ + port = nxt_port_new(task, 1, process->pid, NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + ret = NXT_ERROR; + goto done; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + nxt_queue_insert_tail(&process->ports, &port->link); + + ret = nxt_port_ready_test_queue(thr, task, &msg, port, "first ready"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_ready_test_queue(thr, task, &msg, port, "replacing ready"); + +#endif + +done: + + /* + * The handler leaves the queue mapping and its descriptor on the port, + * and nothing else owns them here: releasing the pool would leak both. + * The fixture port never had a socket pair, so this only frees the queue. + */ + if (port != NULL) { + nxt_port_close(task, port); + } + + thr->runtime = saved_rt; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ready test passed"); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 9edd4c454..ff961e8c4 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -194,6 +194,10 @@ main(int argc, char **argv) return 1; } + if (nxt_port_ready_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CGROUP) if (nxt_cgroup_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 2f77c0706..eab786aa8 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -72,6 +72,7 @@ nxt_int_t nxt_conf_json_depth_test(nxt_thread_t *thr); nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); +nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From ea35f4ae853cf0b275a9ff7175dd0467e7bb4b75 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 21 Aug 2026 23:34:26 +0200 Subject: [PATCH 028/391] port: bound the log file slot a CHANGE_FILE message claims nxt_port_change_log_file_handler() took the slot straight out of the message buffer and handed it to nxt_list_elt(), which returns NULL past the end of the list. The result was dereferenced right away, so a forged CHANGE_FILE naming a slot that does not exist crashes whichever process handles it -- the discovery and prototype processes, the router and the controller all install this handler, and the message is not authenticated. The slot was also read without looking at how much the sender actually wrote: the declared size is what nxt_port_read_msg_process() advances b->mem.free by, so an empty payload reads past the end of the buffer. Bound both, with runtime checks rather than nxt_assert(), which compiles out in release builds -- precisely where an unauthenticated message arrives. Word the diagnostics so that the slot reads as claimed rather than as fact. Neither descriptor of the message was accounted for. The second one is never used on this path yet a message can carry one whatever its type, and the first one survived a failed redirect; the dispatcher does not reclaim what a handler leaves behind, so a peer attaching descriptors to forged messages exhausts the receiver's descriptor table. Close the descriptors on every path that does not consume them, following the convention the PROCESS_READY handler established. The failed-redirect leak lives in nxt_file_redirect() itself, which promises in its own comment that the descriptor is closed and then returns from the dup2() failure without closing it. Callers hand the descriptor over and never close it on success, so nobody did. Close it there instead of guessing in the caller which of the two failures happened -- the close() failure path has released the descriptor already, and closing it a second time could reach whatever reused the number. nxt_main_process_sigusr1_handler() is the other caller and leaked the same way. Co-Authored-By: Claude Opus 5 (1M context) --- auto/sources | 1 + src/nxt_file.c | 11 +- src/nxt_port.c | 70 +++++- src/test/nxt_port_change_file_test.c | 343 +++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 6 files changed, 427 insertions(+), 3 deletions(-) create mode 100644 src/test/nxt_port_change_file_test.c diff --git a/auto/sources b/auto/sources index aa75a7600..f572efa52 100644 --- a/auto/sources +++ b/auto/sources @@ -178,6 +178,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_fail_test.c \ src/test/nxt_port_mmap_range_test.c \ src/test/nxt_port_ready_test.c \ + src/test/nxt_port_change_file_test.c \ " diff --git a/src/nxt_file.c b/src/nxt_file.c index 66edb1adc..d29695e79 100644 --- a/src/nxt_file.c +++ b/src/nxt_file.c @@ -621,7 +621,8 @@ nxt_file_fclose(nxt_task_t *task, FILE *fp) /* * nxt_file_redirect() redirects the file to the fd descriptor. - * Then the fd descriptor is closed. + * Then the fd descriptor is closed -- on the failing paths as well, so that + * the caller can hand the descriptor over unconditionally. */ nxt_int_t @@ -632,6 +633,14 @@ nxt_file_redirect(nxt_file_t *file, nxt_fd_t fd) if (dup2(fd, file->fd) == -1) { nxt_thread_log_alert("dup2(%FD, %FD, \"%FN\") failed %E", fd, file->fd, file->name, nxt_errno); + + /* + * The descriptor is consumed whatever the outcome, as the comment + * above states: callers hand it over and never close it on success, + * so failing without closing it here leaked it instead. + */ + (void) close(fd); + return NXT_ERROR; } diff --git a/src/nxt_port.c b/src/nxt_port.c index 8e398b1d7..09696f964 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -518,7 +518,9 @@ nxt_port_change_log_file(nxt_task_t *task, nxt_runtime_t *rt, nxt_uint_t slot, void nxt_port_change_log_file_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { + size_t size; nxt_buf_t *b; + nxt_int_t ret; nxt_uint_t slot; nxt_file_t *log_file; nxt_runtime_t *rt; @@ -526,18 +528,82 @@ nxt_port_change_log_file_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; b = msg->buf; - slot = *(nxt_uint_t *) b->mem.pos; + /* + * The payload is whatever the sender chose to write: a message with a + * short -- or empty -- buffer would have the slot read past its end. + * The size the sender declared is what nxt_port_read_msg_process() + * advanced b->mem.free by, so the used size is the only bound here. + * nxt_assert() is not used because it compiles out in release builds, + * which is exactly where this message arrives unauthenticated. + */ + size = 0; + + if (nxt_fast_path(b != NULL)) { + size = (size_t) nxt_buf_mem_used_size(&b->mem); + } + + if (nxt_slow_path(size < sizeof(nxt_uint_t))) { + nxt_log(task, NXT_LOG_WARN, "CHANGE_FILE with a %uz byte payload, " + "which is too short to name a log file slot", size); + + nxt_port_recv_msg_close_fds(msg); + return; + } + + /* The payload is not aligned for a nxt_uint_t load. */ + nxt_memcpy(&slot, b->mem.pos, sizeof(nxt_uint_t)); + + /* + * nxt_list_elt() returns NULL past the end of the list, and every + * process that installs this handler -- the discovery and prototype + * processes, the router and the controller -- would then dereference + * it. + */ log_file = nxt_list_elt(rt->log_files, slot); + if (nxt_slow_path(log_file == NULL)) { + nxt_log(task, NXT_LOG_WARN, "CHANGE_FILE claiming log file slot " + "%ui, which does not exist", slot); + + nxt_port_recv_msg_close_fds(msg); + return; + } + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_log(task, NXT_LOG_WARN, "CHANGE_FILE claiming log file slot " + "%ui without a descriptor to redirect it to", slot); + + nxt_port_recv_msg_close_fds(msg); + return; + } + nxt_debug(task, "change log file %FD:%FD", msg->fd[0], log_file->fd); + /* + * A second descriptor is never used on this path, but a message can + * carry one whatever its type: close it before the redirect, so that + * neither outcome of the redirect leaves it behind. + */ + if (msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); + msg->fd[1] = -1; + } + /* * The old log file descriptor must be closed at the moment when no * other threads use it. dup2() allows to use the old file descriptor * for new log file. This change is performed atomically in the kernel. + * + * nxt_file_redirect() consumes the descriptor on every outcome, so drop + * it from the message rather than letting close_fds() reach a number + * that has already been reused. */ - if (nxt_file_redirect(log_file, msg->fd[0]) == NXT_OK) { + ret = nxt_file_redirect(log_file, msg->fd[0]); + + msg->fd[0] = -1; + + if (nxt_fast_path(ret == NXT_OK)) { if (slot == 0) { (void) nxt_file_stderr(log_file); } diff --git a/src/test/nxt_port_change_file_test.c b/src/test/nxt_port_change_file_test.c new file mode 100644 index 000000000..10b67b292 --- /dev/null +++ b/src/test/nxt_port_change_file_test.c @@ -0,0 +1,343 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for nxt_port_change_log_file_handler() (src/nxt_port.c). + * + * CHANGE_FILE carries a log file slot in its payload and the descriptor the + * slot is to be redirected to. Neither is authenticated, and the discovery + * and prototype processes, the router and the controller all install this + * handler. + * + * The handler read the slot without bounding it against the payload, passed + * it to nxt_list_elt() -- which returns NULL past the end of the list -- and + * dereferenced the result, so a forged slot crashed the receiver. It also + * never closed the second descriptor of the message, and lost the first one + * when the redirect failed; the dispatcher does not reclaim fds once the + * handler returns (see nxt_port_recv_msg_close_fds()), so a peer attaching + * descriptors to forged messages can exhaust the receiver's descriptor + * table. + * + * A message can carry two descriptors regardless of how many its type + * normally uses, so the rejected cases attach both and expect both back. + */ + +#include +#include +#include +#include "nxt_tests.h" + +#include + + +static nxt_bool_t +nxt_port_change_file_test_fd_is_open(nxt_fd_t fd) +{ + return fcntl(fd, F_GETFD) != -1; +} + + +/* + * Feed the handler a message it has to refuse, and check that nothing the + * message carried outlives the call. + */ +static nxt_int_t +nxt_port_change_file_test_reject(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_bool_t with_fd0, const char *name) +{ + nxt_fd_t fd0, fd1; + + fd0 = -1; + + if (with_fd0) { + fd0 = open("/dev/null", O_RDONLY); + } + + fd1 = open("/dev/null", O_RDONLY); + + if ((with_fd0 && fd0 == -1) || fd1 == -1) { + nxt_log_alert(thr->log, "port change file test failed to open " + "/dev/null"); + goto fail; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_port_change_log_file_handler(task, msg); + + if ((fd0 != -1 && nxt_port_change_file_test_fd_is_open(fd0)) + || nxt_port_change_file_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port change file test: %s leaked a " + "descriptor", name); + goto fail; /* closes only what is still open */ + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port change file test: %s left fds in the " + "message", name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + /* + * Close only what the handler left open: the descriptors it consumed + * are gone already, and closing them again could reach an unrelated + * descriptor that reused the number. + */ + if (fd0 != -1 && nxt_port_change_file_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_port_change_file_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + return NXT_ERROR; +} + + +/* + * The accepted case: the slot exists, so the first descriptor is redirected + * onto the log file -- and consumed by dup2() doing so -- while the second + * one, which this message type has no use for, is closed rather than left + * behind. + */ +static nxt_int_t +nxt_port_change_file_test_accept(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_file_t *log_file, const char *name) +{ + nxt_fd_t fd0, fd1; + + fd0 = open("/dev/null", O_WRONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (fd0 == -1 || fd1 == -1) { + nxt_log_alert(thr->log, "port change file test failed to open " + "/dev/null"); + goto fail; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_port_change_log_file_handler(task, msg); + + if (nxt_port_change_file_test_fd_is_open(fd0) + || nxt_port_change_file_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port change file test: %s leaked a " + "descriptor", name); + goto fail; + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port change file test: %s left fds in the " + "message", name); + return NXT_ERROR; + } + + if (!nxt_port_change_file_test_fd_is_open(log_file->fd)) { + nxt_log_alert(thr->log, "port change file test: %s closed the log " + "file", name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + if (fd0 != -1 && nxt_port_change_file_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_port_change_file_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + return NXT_ERROR; +} + + +nxt_int_t +nxt_port_change_file_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_buf_t *b; + nxt_file_name_t *name; + nxt_uint_t slot; + nxt_task_t *task; + nxt_int_t ret; + nxt_file_t *log_file; + nxt_runtime_t *rt, *saved_rt; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port change file test started"); + + task = thr->task; + task->thread = thr; + + name = (nxt_file_name_t *) "port change file test"; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + /* + * Two slots. Slot 0 stays unopened, so a redirect onto it fails, and + * the accepted case uses slot 1: a successful redirect of slot 0 makes + * the handler point stderr at the new file, which would follow the test + * process out of this function. + */ + rt->log_files = nxt_list_create(mp, 2, sizeof(nxt_file_t)); + if (nxt_slow_path(rt->log_files == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + log_file = nxt_list_add(rt->log_files); + if (nxt_slow_path(log_file == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(log_file, sizeof(nxt_file_t)); + + /* + * Never opened, so redirecting onto it fails -- and the failure path + * logs the name, which therefore cannot be left NULL. + */ + log_file->name = name; + log_file->fd = NXT_FILE_INVALID; + + log_file = nxt_list_add(rt->log_files); + if (nxt_slow_path(log_file == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(log_file, sizeof(nxt_file_t)); + + log_file->name = name; + log_file->fd = open("/dev/null", O_WRONLY); + if (nxt_slow_path(log_file->fd == -1)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + thr->runtime = rt; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + /* + * No payload at all. The dispatcher always hands the handler a buffer, + * but a NULL one costs a single test to rule out and would be a NULL + * dereference otherwise. + */ + msg.buf = NULL; + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 1, "no payload"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A buffer shorter than the slot it is supposed to carry: the sender + * declares the size, so a zero-length payload arrives the same way a + * full one does, and reading the slot out of it reads past its end. + */ + b = nxt_buf_mem_alloc(mp, sizeof(nxt_uint_t), 0); + if (nxt_slow_path(b == NULL)) { + ret = NXT_ERROR; + goto done; + } + + msg.buf = b; + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 1, + "short payload"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A slot past the end of rt->log_files. nxt_list_elt() returns NULL + * for it, and the handler used to dereference that straight away. + */ + slot = 4242; + b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 1, + "out of range slot"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* An existing slot, but nothing to redirect it to. */ + b->mem.free = b->mem.pos; + slot = 1; + b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 0, + "slot without a descriptor"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A slot whose file was never opened: dup2() onto NXT_FILE_INVALID + * fails, and nxt_file_redirect() used to return without closing the + * descriptor it had been handed -- the caller does not close it on the + * success path, so nobody did. + */ + b->mem.free = b->mem.pos; + slot = 0; + b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); + + ret = nxt_port_change_file_test_reject(thr, task, &msg, 1, + "slot that cannot be redirected"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + b->mem.free = b->mem.pos; + slot = 1; + b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); + + ret = nxt_port_change_file_test_accept(thr, task, &msg, log_file, + "redirected slot"); + +done: + + thr->runtime = saved_rt; + + if (log_file->fd != -1) { + (void) close(log_file->fd); + } + + nxt_mp_destroy(mp); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port change file test passed"); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index ff961e8c4..e50da37f6 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -198,6 +198,10 @@ main(int argc, char **argv) return 1; } + if (nxt_port_change_file_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CGROUP) if (nxt_cgroup_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index eab786aa8..bc2061e52 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -73,6 +73,7 @@ nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); +nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From e21bc5bc50aa1ea647eb5bc84354744d472eb598 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 15:29:06 +0200 Subject: [PATCH 029/391] otel: take the avoidable work off the request path Three changes from #221's cheap tier. They are kept in one commit because they share a single measurement and only one of them earns a number; splitting them would imply three independent wins. == Cache the tracer at init == nxt_otel_rs_get_or_create_trace() called global::tracer_provider().tracer() per request. That takes the global provider's lock, clones the provider and heap-allocates a fresh SdkTracer every time; the SDK does not cache it, and a tracer is meant to be built once. Build it in nxt_otel_rs_init() instead and keep it in a slot beside the provider, behind an RwLock so the request path takes an uncontended read rather than serialising the router threads against each other. The read guard is held across the span build, which is what makes a concurrent reconfigure wait for in-flight builds instead of pulling the tracer out from under them. Neither lock is ever taken while holding the other. An empty slot falls back to the global provider exactly as before, so a span requested before init or after teardown still behaves the way it always has. nxt_otel_rs_shutdown_tracer() clears the slot before the provider goes away. Measured free, not a measured win: on an idle 8c box over 6 interleaved repetitions, the paired difference at sampling_ratio 1.0 -- where this change is the only one of the three that can help -- was -0.55 us +/- 2.28, which is noise. Whatever the per-request tracer costs, it is under the ~2 us this setup can resolve. The change stands on removing an allocation and a global lock from the request path, not on a number. nxt_otel_rs_shutdown_bounded() clears the slot as well. It takes the provider without going through nxt_otel_rs_shutdown_tracer(), so leaving the cache alone there would hand out a tracer whose provider had already been shut down -- an asymmetry that exists only because of this change. Part of #221. == Drop the redundant "Unit Attribute" event == nxt_otel_propagate_header() attached the injected traceparent to the span as a free-form event named "Unit Attribute": two owned Strings, a KeyValue vec and a clock read per request, carrying the trace id, span id and flags that the span's own context already holds and the collector already indexes. Nothing consumes it. The test suite asserts on the semconv attributes, not on events, and has described this one as "the old free-form span events" since they were introduced. It was also being added twice. The event lived in nxt_otel_propagate_header(), not in the attribute stage, and nxt_otel_request_error_path() runs propagate_header() a second time -- so a request that took the error path with a trace it had started itself attached the event, and the traceparent request field, once for each pass. Deleting the event removes that duplication rather than only the allocations. Removes the last caller of nxt_otel_rs_add_event_to_trace(), so the FFI entry point goes with it. Like the tracer caching, this is measured free rather than a measured win: at sampling_ratio 1.0 the two together moved the request path by -0.55 us +/- 2.28 us, i.e. not at all that this setup can see. It stands on the data being redundant. Part of #221. == Skip the attribute work for spans the sampler dropped == set_attribute() on a non-recording span is a no-op, but every argument is built before the call that throws it away: two owned Strings per attribute in Rust, and on the C side the sprintf round-trips, pool allocations and semconv string assembly that produce them. Nothing ever asked whether the span was recording, so all of it ran for spans that would never be exported. That is why lowering sampling_ratio bought back the exporter and nothing on the request path -- measured in #221 as a 12x drop in export-thread CPU with the synchronous cost unchanged inside the noise floor. Measured on an idle 8c box, listen_threads: 1, "return": 200, 100k requests rate-limited to 5k/s, 6 interleaved repetitions with the cell order rotated per repetition, router request-thread CPU only (comm "unitd"; the Rust exporter threads are excluded, so this is the synchronous cost and not the batch export). Paired per-repetition differences: telemetry off -0.52 us (control, t=-0.4, noise) sampling_ratio 1.0 -0.55 us (t=-0.6, noise) sampling_ratio 0.1 -8.87 us (-13.2%, t=-10.6) otel's tax over the same build's telemetry-off cell: before ratio 1.0 18.55 us ratio 0.1 15.45 us (sampling buys 17%) after ratio 1.0 18.52 us ratio 0.1 7.10 us (sampling buys 62%) So the gate removes 54% of otel's request-path cost at ratio 0.1, and by construction nothing at ratio 1.0, where every span records. Ask once, when the span is built, and cache the verdict in nxt_otel_state_t; the three attribute stages consult it instead of calling into Rust again. Propagation stays outside the gate deliberately. W3C Trace Context requires the traceparent to reach the peer and the application whatever the sampling decision was, so that a downstream service can join -- or knowingly decline to join -- the same trace; a sampled-out span still carries a valid trace id and span id, with the sampled flag clear. A new test pins that: with sampling_ratio=0 nothing is exported, and the response still carries a traceparent. It passes before this commit too -- it is there to stop the gate swallowing propagation later. Part of #221. Co-Authored-By: Claude Opus 5 --- src/nxt_otel.c | 60 ++++++++++++++++++------------ src/nxt_otel.h | 5 ++- src/otel/src/lib.rs | 91 ++++++++++++++++++++++++++++++++++----------- test/test_otel.py | 25 +++++++++++++ 4 files changed, 133 insertions(+), 48 deletions(-) diff --git a/src/nxt_otel.c b/src/nxt_otel.c index f6e278bcc..0907f53f7 100644 --- a/src/nxt_otel.c +++ b/src/nxt_otel.c @@ -107,7 +107,6 @@ static void nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) { u_char *traceval; - nxt_str_t traceparent_name, traceparent; nxt_http_field_t *f; traceval = nxt_mp_zalloc(r->mem_pool, NXT_OTEL_TRACEPARENT_LEN + 1); @@ -151,19 +150,6 @@ nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) nxt_http_field_name_set(f, "traceparent"); f->value = traceval; f->value_length = nxt_strlen(traceval); - - traceparent_name = (nxt_str_t) { - .start = f->name, - .length = f->name_length, - }; - - traceparent = (nxt_str_t) { - .start = f->value, - .length = f->value_length, - }; - - nxt_otel_rs_add_event_to_trace(r->otel->trace, - &traceparent_name, &traceparent); } f = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); @@ -180,18 +166,10 @@ nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) static void -nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r) +nxt_otel_span_add_request_attrs(nxt_http_request_t *r) { nxt_str_t val; - nxt_log(task, NXT_LOG_DEBUG, "adding attributes to trace"); - - if (r->otel == NULL || r->otel->trace == NULL) { - nxt_log(task, NXT_LOG_ERR, "no trace to add attributes to!"); - nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); - return; - } - /* * Record only well-defined semconv attributes. We deliberately do NOT * iterate every request header: that would leak sensitive values @@ -229,6 +207,33 @@ nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r) val.length = r->remote->address_length; nxt_otel_add_attr(r, NXT_OTEL_CLIENT_ADDR_TAG, &val); } +} + + +static void +nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r) +{ + nxt_log(task, NXT_LOG_DEBUG, "adding attributes to trace"); + + if (r->otel == NULL || r->otel->trace == NULL) { + nxt_log(task, NXT_LOG_ERR, "no trace to add attributes to!"); + nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE); + return; + } + + /* + * A span the sampler dropped records nothing, yet every attribute value + * is still assembled before the call that discards it. Skip that work. + * + * Propagation below is not part of the bargain: the traceparent must + * reach the peer and the application whatever the sampling decision was, + * so that a downstream service can continue -- or deliberately not + * continue -- the same trace. + */ + + if (r->otel->recording) { + nxt_otel_span_add_request_attrs(r); + } nxt_otel_propagate_header(task, r); @@ -245,6 +250,11 @@ nxt_otel_span_add_body(nxt_http_request_t *r) nxt_int_t cur; nxt_str_t body_val; + if (!r->otel->recording) { + nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE); + return; + } + if (r->body != NULL) { body_size = nxt_buf_used_size(r->body); } @@ -291,7 +301,7 @@ nxt_otel_span_add_status(nxt_task_t *task, nxt_http_request_t *r) nxt_app_t *app; nxt_request_rpc_data_t *rpc; - if (r->otel == NULL || r->otel->trace == NULL) { + if (r->otel == NULL || r->otel->trace == NULL || !r->otel->recording) { return; } @@ -506,6 +516,8 @@ nxt_otel_trace_and_span_init(nxt_task_t *task, nxt_http_request_t *r) return; } + r->otel->recording = nxt_otel_rs_is_recording(r->otel->trace); + nxt_otel_state_transition(r->otel, NXT_OTEL_HEADER_STATE); } diff --git a/src/nxt_otel.h b/src/nxt_otel.h index 42c5f640c..34f81e654 100644 --- a/src/nxt_otel.h +++ b/src/nxt_otel.h @@ -26,10 +26,9 @@ extern void nxt_otel_rs_init( const nxt_str_t *endpoint, const nxt_str_t *protocol, double sample_fraction, double batch_size); extern void nxt_otel_rs_copy_traceparent(u_char *buffer, void *span); -extern void nxt_otel_rs_add_event_to_trace(void *trace, nxt_str_t *key, - nxt_str_t *val); extern void nxt_otel_rs_add_attr(void *trace, nxt_str_t *key, nxt_str_t *val); extern void nxt_otel_rs_set_error(void *trace); +extern uint8_t nxt_otel_rs_is_recording(void *trace); extern uint8_t nxt_otel_rs_is_init(void); extern void nxt_otel_rs_uninit(void); extern uint8_t nxt_otel_rs_shutdown_bounded(uint64_t timeout_ms); @@ -90,6 +89,8 @@ struct nxt_otel_state_s { void *trace; nxt_otel_status_t status; nxt_str_t trace_state; + /* The sampler's verdict on this span, cached at creation. */ + uint8_t recording; }; diff --git a/src/otel/src/lib.rs b/src/otel/src/lib.rs index c6c1be746..705fa7e41 100644 --- a/src/otel/src/lib.rs +++ b/src/otel/src/lib.rs @@ -14,7 +14,7 @@ //! which exports it from its own background thread. use opentelemetry::global; -use opentelemetry::global::BoxedSpan; +use opentelemetry::global::{BoxedSpan, BoxedTracer}; use opentelemetry::trace::{ Span, SpanContext, SpanId, SpanKind, Status, TraceContextExt, TraceFlags, TraceId, TraceState, Tracer, TracerProvider, @@ -31,7 +31,7 @@ use std::ffi::{c_char, CStr, CString}; use std::str::FromStr; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc; -use std::sync::Mutex; +use std::sync::{Mutex, RwLock}; use std::time::Duration; use std::{ptr, slice}; @@ -73,6 +73,19 @@ fn provider_slot() -> &'static Mutex> { &PROVIDER } +/// The tracer built from the live provider. +/// +/// `global::tracer_provider().tracer()` takes the global provider's lock, +/// clones the provider and heap-allocates a fresh tracer on every call, but a +/// tracer is meant to be built once. Caching it leaves the request path with +/// one uncontended read lock instead. `None` means no provider is installed; +/// the request path then falls back to the global one, so behaviour is +/// unchanged either way. +fn tracer_slot() -> &'static RwLock> { + static TRACER: RwLock> = RwLock::new(None); + &TRACER +} + /// The tokio runtime owning the gRPC exporter's tonic channel. The blocking /// batch processor drives export RPCs onto it from its own thread, so it must /// outlive the provider; it is dropped in `nxt_otel_rs_shutdown_tracer`. @@ -341,6 +354,10 @@ pub unsafe extern "C" fn nxt_otel_rs_init( global::set_tracer_provider(provider.clone()); + if let Ok(mut slot) = tracer_slot().write() { + *slot = Some(global::tracer_provider().tracer(TRACER_NAME)); + } + if let Ok(mut slot) = provider_slot().lock() { *slot = Some(provider); } @@ -372,8 +389,10 @@ pub unsafe extern "C" fn nxt_otel_rs_copy_traceparent(buf: *mut c_char, span: *c *buf.add(TRACEPARENT_HEADER_LEN as usize) = 0; } +/// Set a semantic-convention span attribute (e.g. `http.request.method`). +/// These are structured span attributes the collector can index and query. #[no_mangle] -pub unsafe extern "C" fn nxt_otel_rs_add_event_to_trace( +pub unsafe extern "C" fn nxt_otel_rs_add_attr( trace: *mut BoxedSpan, key: *const nxt_str_t, val: *const nxt_str_t, @@ -385,26 +404,23 @@ pub unsafe extern "C" fn nxt_otel_rs_add_event_to_trace( let key = nxt_str_to_string(&*key); let val = nxt_str_to_string(&*val); - (*trace).add_event("Unit Attribute".to_string(), vec![KeyValue::new(key, val)]); + (*trace).set_attribute(KeyValue::new(key, val)); } -/// Set a semantic-convention span attribute (e.g. `http.request.method`). -/// Unlike `add_event_to_trace`, this records structured span attributes the -/// collector can index and query, not timestamped events. +/// Whether the sampler kept this span. +/// +/// C asks once, right after the span is built, and skips the attribute work +/// for a span that is not recording. `set_attribute` on such a span is +/// already a no-op, but its arguments are built before the call that throws +/// them away -- so without this gate, lowering `sampling_ratio` buys back the +/// exporter and nothing on the request path. #[no_mangle] -pub unsafe extern "C" fn nxt_otel_rs_add_attr( - trace: *mut BoxedSpan, - key: *const nxt_str_t, - val: *const nxt_str_t, -) { - if trace.is_null() || key.is_null() || val.is_null() { - return; +pub unsafe extern "C" fn nxt_otel_rs_is_recording(trace: *mut BoxedSpan) -> u8 { + if trace.is_null() { + return 0; } - let key = nxt_str_to_string(&*key); - let val = nxt_str_to_string(&*val); - - (*trace).set_attribute(KeyValue::new(key, val)); + (*trace).is_recording() as u8 } /// Mark the span as errored. Called by C for 5xx responses so the trace is @@ -457,6 +473,12 @@ unsafe fn nxt_otel_parent_context( Some(Context::new().with_remote_span_context(sc)) } +fn nxt_otel_build_span(tracer: &BoxedTracer, parent: &Context) -> BoxedSpan { + let builder = tracer.span_builder(SPAN_NAME).with_kind(SpanKind::Server); + + tracer.build_with_context(builder, parent) +} + #[no_mangle] pub unsafe extern "C" fn nxt_otel_rs_get_or_create_trace( trace_id: *const c_char, @@ -464,12 +486,23 @@ pub unsafe extern "C" fn nxt_otel_rs_get_or_create_trace( trace_flags: *const c_char, trace_state: *const nxt_str_t, ) -> *mut BoxedSpan { - let tracer = global::tracer_provider().tracer(TRACER_NAME); - let builder = tracer.span_builder(SPAN_NAME).with_kind(SpanKind::Server); - let parent = nxt_otel_parent_context(trace_id, parent_id, trace_flags, trace_state) .unwrap_or_else(Context::new); - let span = tracer.build_with_context(builder, &parent); + + // The read guard is held across the build so the cached tracer cannot be + // dropped by a concurrent reconfigure while a span is being made from it. + let cached = tracer_slot().read().ok(); + + let span = match cached.as_ref().and_then(|slot| slot.as_ref()) { + Some(tracer) => nxt_otel_build_span(tracer, &parent), + None => { + // No provider installed, or the lock is poisoned: fall back to the + // global provider, which hands out a no-op tracer in that case. + let tracer = global::tracer_provider().tracer(TRACER_NAME); + + nxt_otel_build_span(&tracer, &parent) + } + }; Box::into_raw(Box::new(span)) } @@ -524,6 +557,14 @@ pub unsafe extern "C" fn nxt_otel_rs_send_trace(trace: *mut BoxedSpan) { /// export has failed", not "no spans were lost". #[no_mangle] pub unsafe extern "C" fn nxt_otel_rs_shutdown_bounded(timeout_ms: u64) -> u8 { + // This path takes the provider without going through + // nxt_otel_rs_shutdown_tracer(), so the cached tracer has to be dropped + // here too: leaving it would hand out a tracer whose provider has + // already been shut down. + if let Ok(mut slot) = tracer_slot().write() { + *slot = None; + } + let provider = provider_slot().lock().ok().and_then(|mut g| g.take()); let rt = runtime_slot().lock().ok().and_then(|mut g| g.take()); @@ -582,6 +623,12 @@ pub unsafe extern "C" fn nxt_otel_rs_shutdown_bounded(timeout_ms: u64) -> u8 { /// Flush and tear down the live tracer provider, if any. #[no_mangle] pub unsafe extern "C" fn nxt_otel_rs_shutdown_tracer() { + // Dropped first: a tracer handed out after this point would belong to a + // provider that is already being torn down. + if let Ok(mut slot) = tracer_slot().write() { + *slot = None; + } + let provider = provider_slot().lock().ok().and_then(|mut g| g.take()); if let Some(provider) = provider { // Flushes pending spans; on a grpc build this still needs the runtime, diff --git a/test/test_otel.py b/test/test_otel.py index f7b607f39..e9b84f76b 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -312,6 +312,31 @@ def test_otel_sampling_zero_exports_nothing(protocol): _kill(proc) +@_skipif_no_fake_otlp +@pytest.mark.parametrize('protocol', ['http', 'grpc']) +def test_otel_sampling_zero_still_propagates(protocol): + """sampling_ratio=0 → nothing is exported, but context still propagates. + + The request path skips the attribute work for a span the sampler dropped + (nxt_otel_span_add_headers, src/nxt_otel.c). Propagation sits outside that + gate on purpose: W3C Trace Context requires the traceparent to reach the + peer and the application whatever the sampling decision was, so a + downstream service can join -- or knowingly decline to join -- the trace. + """ + port = _get_free_port() + proc = _run_fake_otlp(port, protocol=protocol) # run forever, absorb exports + try: + _configure_or_skip(port, sampling_ratio=0.0, protocol=protocol) + + resp = _get_until_header('traceparent') + assert resp['status'] == 200 + assert 'traceparent' in _response_headers_lower(resp), ( + 'an unsampled request must still carry a traceparent header' + ) + finally: + _kill(proc) + + @_skipif_no_fake_otlp @pytest.mark.parametrize('protocol', ['http', 'grpc']) def test_otel_traceparent_malformed(protocol): From defd7e2665bf9e5fa63a94eab0f083d137343c65 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 15:31:43 +0200 Subject: [PATCH 030/391] otel: static attribute keys, integer values, one crossing per stage Issue #221 measured a traced request at ~13 C->Rust FFI crossings and ~25-30 heap allocations. The gate and the cached tracer removed the cost that sampling should have removed; this removes the cost that remains when sampling_ratio is 1.0, where every request is recorded and the attribute path runs in full. Three changes that only make sense together: Keys become ids. All eleven attribute keys are compile-time constants that were being strlen'd and copied onto the heap on every request. They now live in a static ATTR_KEYS table of Key::from_static_str on the Rust side, indexed by nxt_otel_attr_id_t. The [Key; NXT_OTEL_ATTR_MAX] annotation makes a mismatched element count a compile error; the C enum stays a reviewed contract, as the shutdown constants already do. Integers travel as integers. http.response.status_code and http.request.body.size are Value::I64. The semantic conventions type them that way, and it deletes both string round-trips: the snprintf and status_buf in the status stage, and the whole log10/sprintf/nxt_mp_alloc block in the body stage -- and math.h with it. Attributes cross once per stage. nxt_otel_rs_add_attrs takes a stack-built array, so the request stage goes from eight FFI crossings to one and the status stage from three to one. The single-attribute entry point had no other callers and is gone. The saving is the crossings and the keys, not batching inside the SDK. An earlier version of this collected the stage's attributes into a Vec::with_capacity(n) and called set_attributes once; that was measurably the wrong shape. BoxedSpan does not override Span::set_attributes, so it takes the default trait implementation, which loops over set_attribute anyway -- the Vec was a per-stage allocation the code it replaced did not make. Do not reintroduce it: setting each attribute directly in the loop is deliberate, not an oversight. Note this deliberately does not use SpanBuilder::with_attributes, which issue #221 suggests. That assembles attributes while the span is built, before C can observe the sampling decision, which would undo the is_recording gate that is the measured win. Attributes are set after the gate, on a span already known to be recording. One state-machine consequence worth naming, because it is only visible when this change and the gate are read together: nxt_otel_span_add_body used to return without transitioning if nxt_mp_alloc failed. There is no allocation left on that path, and the gate's early return transitions too, so every exit from that function now transitions exactly once. test_otel_span_5xx_status could not survive the typing change: it scanned for the ASCII bytes "503", which an int_value varint does not contain. It now asserts the AnyValue int_value encoding next to the status_code key, scanning every occurrence because the readiness spans carry int_value 200 and appear first. That proves the semconv typing rather than proving some "503" exists somewhere in the payload. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013uPosiJqof76yNnasGdgFn --- src/nxt_otel.c | 208 +++++++++++++++++++++++++------------------- src/nxt_otel.h | 45 +++++++++- src/otel/src/lib.rs | 105 +++++++++++++++++++--- test/test_otel.py | 75 +++++++++++++--- 4 files changed, 321 insertions(+), 112 deletions(-) diff --git a/src/nxt_otel.c b/src/nxt_otel.c index 0907f53f7..99a11cbcc 100644 --- a/src/nxt_otel.c +++ b/src/nxt_otel.c @@ -3,8 +3,6 @@ * Copyright (C) F5, Inc. */ -#include - #include #include #include @@ -22,24 +20,6 @@ #define NXT_OTEL_TRACEPARENT_LEN 55 -/* - * Span attribute keys. These follow the stable OpenTelemetry HTTP semantic - * conventions (https://opentelemetry.io/docs/specs/semconv/http/http-spans/); - * "http.flavor"/"http.user_agent" from older drafts are superseded by - * "network.protocol.version"/"user_agent.original". "unit.application.*" are - * FreeUnit-specific: a reverse proxy can't know the served app, but Unit can. - */ -#define NXT_OTEL_BODY_SIZE_TAG "http.request.body.size" -#define NXT_OTEL_METHOD_TAG "http.request.method" -#define NXT_OTEL_PATH_TAG "url.path" -#define NXT_OTEL_SCHEME_TAG "url.scheme" -#define NXT_OTEL_FLAVOR_TAG "network.protocol.version" -#define NXT_OTEL_USER_AGENT_TAG "user_agent.original" -#define NXT_OTEL_SERVER_ADDR_TAG "server.address" -#define NXT_OTEL_CLIENT_ADDR_TAG "client.address" -#define NXT_OTEL_APP_NAME_TAG "unit.application.name" -#define NXT_OTEL_APP_TYPE_TAG "unit.application.type" -#define NXT_OTEL_STATUS_CODE_TAG "http.response.status_code" static void @@ -54,27 +34,111 @@ nxt_otel_state_transition(nxt_otel_state_t *state, nxt_otel_status_t status) /* - * Set a semantic-convention span attribute from a string-literal key and an - * nxt_str_t value. Empty or absent values are skipped so we don't emit blank - * attributes for headers the request didn't carry. + * A request's span attributes, accumulated on the stack and handed to Rust in + * one call. + * + * Each attribute used to cost its own FFI crossing and two heap allocations -- + * one for the key, one for the value -- and the keys were compile-time + * constants being strlen'd and copied per request. Now the key is an id into a + * static table on the Rust side, and a whole stage's attributes cross once. + * + * The batch never outlives the stage that fills it, so the nxt_str_t values it + * holds are borrowed, not copied: they stay valid until the flush, and Rust + * copies what it needs to keep. + */ +typedef struct { + nxt_otel_attr_t attrs[NXT_OTEL_ATTR_MAX]; + nxt_uint_t n; +} nxt_otel_attr_batch_t; + + +static void +nxt_otel_attr_batch_init(nxt_otel_attr_batch_t *batch) +{ + batch->n = 0; +} + + +static nxt_otel_attr_t * +nxt_otel_attr_next(nxt_otel_attr_batch_t *batch, nxt_otel_attr_id_t id) +{ + nxt_otel_attr_t *attr; + + /* + * Every stage adds at most one attribute per id, so the batch cannot + * overflow; the check is here so that a future caller adding a second + * attribute for one id truncates rather than writing off the end. + */ + if (nxt_slow_path(batch->n >= NXT_OTEL_ATTR_MAX)) { + return NULL; + } + + attr = &batch->attrs[batch->n++]; + attr->key_id = id; + + return attr; +} + + +/* + * Add a string attribute. Empty or absent values are skipped so we don't emit + * blank attributes for headers the request didn't carry. */ static void -nxt_otel_add_attr(nxt_http_request_t *r, const char *key, nxt_str_t *val) +nxt_otel_attr_str(nxt_otel_attr_batch_t *batch, nxt_otel_attr_id_t id, + nxt_str_t *val) { - nxt_str_t k; + nxt_otel_attr_t *attr; if (val == NULL || val->start == NULL || val->length == 0) { return; } - if (r->otel == NULL || r->otel->trace == NULL) { + attr = nxt_otel_attr_next(batch, id); + if (attr == NULL) { return; } - k.start = (u_char *) key; - k.length = nxt_strlen(key); + attr->type = NXT_OTEL_ATTR_TYPE_STR; + attr->ival = 0; + attr->sval = *val; +} + + +/* + * Add an integer attribute. The semantic conventions type these as integers, + * and passing one as an integer avoids a sprintf round-trip on the request + * path as well as being the correct value type on the wire. + */ +static void +nxt_otel_attr_i64(nxt_otel_attr_batch_t *batch, nxt_otel_attr_id_t id, + int64_t val) +{ + nxt_otel_attr_t *attr; - nxt_otel_rs_add_attr(r->otel->trace, &k, val); + attr = nxt_otel_attr_next(batch, id); + if (attr == NULL) { + return; + } + + attr->type = NXT_OTEL_ATTR_TYPE_I64; + attr->ival = val; + nxt_memzero(&attr->sval, sizeof(nxt_str_t)); +} + + +static void +nxt_otel_attr_flush(nxt_http_request_t *r, nxt_otel_attr_batch_t *batch) +{ + if (batch->n == 0) { + return; + } + + if (r->otel == NULL || r->otel->trace == NULL) { + return; + } + + nxt_otel_rs_add_attrs(r->otel->trace, batch->attrs, batch->n); } @@ -168,21 +232,24 @@ nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) static void nxt_otel_span_add_request_attrs(nxt_http_request_t *r) { - nxt_str_t val; + nxt_str_t val; + nxt_otel_attr_batch_t batch; /* * Record only well-defined semconv attributes. We deliberately do NOT * iterate every request header: that would leak sensitive values * (Authorization, Cookie, ...) into the telemetry backend. */ - nxt_otel_add_attr(r, NXT_OTEL_METHOD_TAG, r->method); - nxt_otel_add_attr(r, NXT_OTEL_PATH_TAG, r->path); + nxt_otel_attr_batch_init(&batch); + + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_METHOD, r->method); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_PATH, r->path); nxt_str_set(&val, "http"); if (r->tls) { nxt_str_set(&val, "https"); } - nxt_otel_add_attr(r, NXT_OTEL_SCHEME_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_SCHEME, &val); /* "HTTP/1.1" -> "1.1" for network.protocol.version */ val = r->version; @@ -192,21 +259,23 @@ nxt_otel_span_add_request_attrs(nxt_http_request_t *r) val.start += nxt_length("HTTP/"); val.length -= nxt_length("HTTP/"); } - nxt_otel_add_attr(r, NXT_OTEL_FLAVOR_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_FLAVOR, &val); if (r->user_agent != NULL) { val.start = r->user_agent->value; val.length = r->user_agent->value_length; - nxt_otel_add_attr(r, NXT_OTEL_USER_AGENT_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_USER_AGENT, &val); } - nxt_otel_add_attr(r, NXT_OTEL_SERVER_ADDR_TAG, &r->host); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_SERVER_ADDR, &r->host); if (r->remote != NULL) { val.start = nxt_sockaddr_address(r->remote); val.length = r->remote->address_length; - nxt_otel_add_attr(r, NXT_OTEL_CLIENT_ADDR_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_CLIENT_ADDR, &val); } + + nxt_otel_attr_flush(r, &batch); } @@ -244,11 +313,8 @@ nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r) static void nxt_otel_span_add_body(nxt_http_request_t *r) { - size_t body_size = 0; - size_t buf_size; - u_char *body_buf, *body_size_buf; - nxt_int_t cur; - nxt_str_t body_val; + size_t body_size = 0; + nxt_otel_attr_batch_t batch; if (!r->otel->recording) { nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE); @@ -259,34 +325,10 @@ nxt_otel_span_add_body(nxt_http_request_t *r) body_size = nxt_buf_used_size(r->body); } - buf_size = 1; // first digit - if (body_size != 0) { - buf_size += log10(body_size); // subsequent digits - } - buf_size += 1; // \0 - buf_size += nxt_strlen(NXT_OTEL_BODY_SIZE_TAG); - buf_size += 1; // \0 - - body_buf = nxt_mp_alloc(r->mem_pool, buf_size); - if (nxt_slow_path(body_buf == NULL)) { - return; - } - - cur = sprintf((char *) body_buf, "%lu", body_size); - if (cur < 0) { - return; - } + nxt_otel_attr_batch_init(&batch); + nxt_otel_attr_i64(&batch, NXT_OTEL_ATTR_BODY_SIZE, (int64_t) body_size); + nxt_otel_attr_flush(r, &batch); - cur += 1; - body_size_buf = body_buf + cur; - nxt_cpystr(body_buf + cur, (const u_char *) NXT_OTEL_BODY_SIZE_TAG); - - body_val = (nxt_str_t) { - .start = body_buf, - .length = nxt_strlen(body_buf), - }; - - nxt_otel_add_attr(r, (const char *) body_size_buf, &body_val); nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE); } @@ -294,17 +336,18 @@ nxt_otel_span_add_body(nxt_http_request_t *r) static void nxt_otel_span_add_status(nxt_task_t *task, nxt_http_request_t *r) { - int n; - u_char status_buf[8]; const char *type_name; nxt_str_t val; nxt_app_t *app; + nxt_otel_attr_batch_t batch; nxt_request_rpc_data_t *rpc; if (r->otel == NULL || r->otel->trace == NULL || !r->otel->recording) { return; } + nxt_otel_attr_batch_init(&batch); + /* * Application identity is resolved during routing, so it is only known by * the time the span is collected (after the response). A reverse proxy @@ -313,32 +356,21 @@ nxt_otel_span_add_status(nxt_task_t *task, nxt_http_request_t *r) rpc = r->req_rpc_data; if (rpc != NULL && rpc->app != NULL) { app = rpc->app; - nxt_otel_add_attr(r, NXT_OTEL_APP_NAME_TAG, &app->name); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_APP_NAME, &app->name); type_name = nxt_otel_app_type_name(app->type); val.start = (u_char *) type_name; val.length = nxt_strlen(type_name); - nxt_otel_add_attr(r, NXT_OTEL_APP_TYPE_TAG, &val); + nxt_otel_attr_str(&batch, NXT_OTEL_ATTR_APP_TYPE, &val); } // dont bother logging an unset status - if (r->status == 0) { - return; + if (r->status != 0) { + nxt_otel_attr_i64(&batch, NXT_OTEL_ATTR_STATUS_CODE, + (int64_t) r->status); } - n = snprintf((char *) status_buf, sizeof(status_buf), "%d", - (int) r->status); - /* - * snprintf() returns the length it *would* have written; on truncation - * that exceeds the buffer, so only record the attribute when the value - * fully fit -- a truncated status code is meaningless anyway, and using - * the would-have-been length as val.length would read past status_buf. - */ - if (n > 0 && n < (int) sizeof(status_buf)) { - val.start = status_buf; - val.length = (size_t) n; - nxt_otel_add_attr(r, NXT_OTEL_STATUS_CODE_TAG, &val); - } + nxt_otel_attr_flush(r, &batch); /* Flag server errors so the span shows Status::Error in the collector. */ if (r->status >= NXT_HTTP_INTERNAL_SERVER_ERROR) { diff --git a/src/nxt_otel.h b/src/nxt_otel.h index 34f81e654..52a9ee14f 100644 --- a/src/nxt_otel.h +++ b/src/nxt_otel.h @@ -16,6 +16,48 @@ #endif +/* + * Span attribute keys, as ids rather than strings. + * + * The key text lives on the Rust side in ATTR_KEYS, as a const table of + * Key::from_static_str -- so a key costs no allocation and no strlen per + * request. The order of this enum and of that table is the contract between + * the two sides: the id indexes the table directly. Add to the end, and add + * to ATTR_KEYS in src/otel/src/lib.rs in the same commit. + */ +typedef enum { + NXT_OTEL_ATTR_METHOD = 0, + NXT_OTEL_ATTR_PATH, + NXT_OTEL_ATTR_SCHEME, + NXT_OTEL_ATTR_FLAVOR, + NXT_OTEL_ATTR_USER_AGENT, + NXT_OTEL_ATTR_SERVER_ADDR, + NXT_OTEL_ATTR_CLIENT_ADDR, + NXT_OTEL_ATTR_APP_NAME, + NXT_OTEL_ATTR_APP_TYPE, + NXT_OTEL_ATTR_STATUS_CODE, + NXT_OTEL_ATTR_BODY_SIZE, + NXT_OTEL_ATTR_MAX +} nxt_otel_attr_id_t; + + +/* Which member of nxt_otel_attr_t carries the value. */ +#define NXT_OTEL_ATTR_TYPE_STR 0 +#define NXT_OTEL_ATTR_TYPE_I64 1 + + +/* + * One span attribute in a batch. Mirrored by nxt_otel_attr_t in + * src/otel/src/lib.rs; the layouts must stay identical. + */ +typedef struct { + uint32_t key_id; + uint32_t type; + int64_t ival; + nxt_str_t sval; +} nxt_otel_attr_t; + + #if (NXT_HAVE_OTEL) extern void nxt_otel_rs_send_trace(void *trace); extern void * nxt_otel_rs_get_or_create_trace(const u_char *trace_id, @@ -26,7 +68,8 @@ extern void nxt_otel_rs_init( const nxt_str_t *endpoint, const nxt_str_t *protocol, double sample_fraction, double batch_size); extern void nxt_otel_rs_copy_traceparent(u_char *buffer, void *span); -extern void nxt_otel_rs_add_attr(void *trace, nxt_str_t *key, nxt_str_t *val); +extern void nxt_otel_rs_add_attrs(void *trace, + const nxt_otel_attr_t *attrs, size_t n); extern void nxt_otel_rs_set_error(void *trace); extern uint8_t nxt_otel_rs_is_recording(void *trace); extern uint8_t nxt_otel_rs_is_init(void); diff --git a/src/otel/src/lib.rs b/src/otel/src/lib.rs index 705fa7e41..5f2a54526 100644 --- a/src/otel/src/lib.rs +++ b/src/otel/src/lib.rs @@ -19,7 +19,7 @@ use opentelemetry::trace::{ Span, SpanContext, SpanId, SpanKind, Status, TraceContextExt, TraceFlags, TraceId, TraceState, Tracer, TracerProvider, }; -use opentelemetry::{Context, KeyValue}; +use opentelemetry::{Context, Key, KeyValue, Value}; use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig}; use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::trace::{ @@ -52,6 +52,55 @@ const NXT_OTEL_SHUTDOWN_TIMEOUT: u8 = 0; const NXT_OTEL_SHUTDOWN_FLUSHED: u8 = 1; const NXT_OTEL_SHUTDOWN_FAILED: u8 = 2; +/// Span attribute keys, indexed by the `nxt_otel_attr_id_t` id C sends. +/// +/// `Key::from_static_str` is a `const fn` over a `&'static str`, so a key +/// costs no allocation and no `strlen` -- where the previous string-key FFI +/// copied all eleven of these compile-time constants onto the heap on every +/// traced request. +/// +/// This table and `nxt_otel_attr_id_t` in `src/nxt_otel.h` are one contract: +/// the id indexes this array directly, so the order must match exactly. +/// The `[Key; NXT_OTEL_ATTR_MAX]` annotation makes a mismatched element count +/// a compile error, so a key added here without extending the count cannot +/// ship. It cannot check the C enum itself -- the ids stay a reviewed +/// contract -- but it catches the half of the mistake that lives on this side. +/// +/// These follow the stable OpenTelemetry HTTP semantic conventions +/// (); +/// "http.flavor"/"http.user_agent" from older drafts are superseded by +/// "network.protocol.version"/"user_agent.original". "unit.application.*" are +/// FreeUnit-specific: a reverse proxy can't know the served app, but Unit can. +static ATTR_KEYS: [Key; NXT_OTEL_ATTR_MAX] = [ + Key::from_static_str("http.request.method"), + Key::from_static_str("url.path"), + Key::from_static_str("url.scheme"), + Key::from_static_str("network.protocol.version"), + Key::from_static_str("user_agent.original"), + Key::from_static_str("server.address"), + Key::from_static_str("client.address"), + Key::from_static_str("unit.application.name"), + Key::from_static_str("unit.application.type"), + Key::from_static_str("http.response.status_code"), + Key::from_static_str("http.request.body.size"), +]; + +/// Must equal `NXT_OTEL_ATTR_MAX` in `src/nxt_otel.h`. +const NXT_OTEL_ATTR_MAX: usize = 11; + +const NXT_OTEL_ATTR_TYPE_STR: u32 = 0; +const NXT_OTEL_ATTR_TYPE_I64: u32 = 1; + +/// One span attribute in a batch. Mirrors `nxt_otel_attr_t` in +/// `src/nxt_otel.h`; the layouts must stay identical. +#[repr(C)] +pub struct nxt_otel_attr_t { + pub key_id: u32, + pub r#type: u32, + pub ival: i64, + pub sval: nxt_str_t, +} + #[repr(C)] pub struct nxt_str_t { pub length: usize, @@ -389,22 +438,58 @@ pub unsafe extern "C" fn nxt_otel_rs_copy_traceparent(buf: *mut c_char, span: *c *buf.add(TRACEPARENT_HEADER_LEN as usize) = 0; } -/// Set a semantic-convention span attribute (e.g. `http.request.method`). -/// These are structured span attributes the collector can index and query. +/// Set a stage's semantic-convention span attributes in one call. +/// +/// C accumulates a stage's attributes on its stack and crosses once, rather +/// than once per attribute: a traced request used to make eight or nine FFI +/// crossings here and allocate two `String`s at each of them. Only the values +/// still allocate, and only those that are strings. +/// +/// Note this deliberately does NOT use `SpanBuilder::with_attributes`, which +/// issue #221 suggests. That would assemble the attributes while the span is +/// being built -- before C can observe the sampling decision -- which would +/// undo the `is_recording` gate that is the measured win. Attributes are set +/// after the gate, on a span already known to be recording. +/// +/// # Safety +/// +/// `attrs` must point at `n` initialised `nxt_otel_attr_t`, and each entry's +/// `sval` must reference bytes that stay valid for the duration of the call. #[no_mangle] -pub unsafe extern "C" fn nxt_otel_rs_add_attr( +pub unsafe extern "C" fn nxt_otel_rs_add_attrs( trace: *mut BoxedSpan, - key: *const nxt_str_t, - val: *const nxt_str_t, + attrs: *const nxt_otel_attr_t, + n: usize, ) { - if trace.is_null() || key.is_null() || val.is_null() { + if trace.is_null() || attrs.is_null() || n == 0 { return; } - let key = nxt_str_to_string(&*key); - let val = nxt_str_to_string(&*val); + let attrs = slice::from_raw_parts(attrs, n); + + // Set each attribute directly rather than collecting into a Vec first. + // `Span::set_attributes` is a default trait method that just loops over + // `set_attribute`, and `BoxedSpan` does not override it, so a Vec here + // would be an allocation that buys nothing. The saving this function + // exists for is the single FFI crossing and the static keys, not batching + // inside the SDK. + for attr in attrs { + // A key_id C and Rust disagree about would index out of bounds; + // drop the attribute rather than panic across the FFI boundary. + let Some(key) = ATTR_KEYS.get(attr.key_id as usize) else { + continue; + }; - (*trace).set_attribute(KeyValue::new(key, val)); + let value = match attr.r#type { + NXT_OTEL_ATTR_TYPE_I64 => Value::I64(attr.ival), + NXT_OTEL_ATTR_TYPE_STR => Value::String(nxt_str_to_string(&attr.sval).into()), + _ => continue, + }; + + // Cloning a Key built by `from_static_str` copies a &'static str -- + // no allocation, which is the point of the ATTR_KEYS table. + (*trace).set_attribute(KeyValue::new(key.clone(), value)); + } } /// Whether the sampler kept this span. diff --git a/test/test_otel.py b/test/test_otel.py index e9b84f76b..a526b226e 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -118,6 +118,41 @@ def _require_otel(): _configure_or_skip(1) +STATUS_CODE_KEY = b'http.response.status_code' + + +def _otlp_int_value(value): + """Encode an OTLP AnyValue int_value as it appears on the wire. + + int_value is field 3 of AnyValue with varint wire type, so the tag byte is + (3 << 3) | 0 == 0x18, followed by the value as a base-128 varint. + """ + out = bytearray(b'\x18') + while True: + byte = value & 0x7F + value >>= 7 + out.append(byte | 0x80 if value else byte) + if not value: + return bytes(out) + + +def _has_keyed_int_attr(body, key, value): + """Whether `key` is followed by an int_value attribute equal to `value`. + + The dump holds raw trace ids, span ids and timestamps, so the encoded value + can occur anywhere by coincidence. Anchoring the search to the bytes just + after each occurrence of the key is what makes this a statement about the + attribute rather than about the payload. + """ + want = _otlp_int_value(value) + at = body.find(key) + while at != -1: + if want in body[at + len(key):at + len(key) + 16]: + return True + at = body.find(key, at + 1) + return False + + def _response_headers_lower(resp): return {k.lower(): v for k, v in resp['headers'].items()} @@ -182,11 +217,15 @@ def test_otel_span_exported_with_service_name(tmp_path, protocol): def test_otel_span_5xx_status(tmp_path, protocol): """A 5xx response is still traced and the span records the error status. - The status_code attribute is emitted as a string (sprintf "%d"), so the - literal `503` travels in the OTLP payload. Per the 1.35.6 fix, 503 >= 500 - also marks the span Status::Error (nxt_otel_rs_set_error); that enum is not - asserted here (it is not a literal in the protobuf), but the status_code - value is the concrete, reliable signal. + http.response.status_code is an OTLP *integer* attribute, so 503 travels as + a varint rather than the ASCII bytes `503`: in AnyValue, int_value is field + 3, giving tag 0x18 followed by varint 503 (0xf7 0x03). Asserting on those + bytes is what distinguishes a correctly typed integer attribute from the + stringified one the semantic conventions do not ask for. + + Per the 1.35.6 fix, 503 >= 500 also marks the span Status::Error + (nxt_otel_rs_set_error); that enum is not asserted here (it is a protobuf + enum, not a literal). """ port = _get_free_port() dump = str(tmp_path / 'otlp_dump.bin') @@ -208,11 +247,11 @@ def test_otel_span_5xx_status(tmp_path, protocol): '503', 'routes/0/action/return' ), 'switch route to return 503' - # Fire 503s until the error span reaches the collector. status_code is - # emitted as the string "503" (sprintf "%d"); per the 1.35.6 fix, - # 503 >= 500 also marks the span Status::Error (not asserted here -- it - # is a protobuf enum, not a literal). Only the 503 requests can add the - # "503" bytes, so the readiness 200 spans don't false-positive. + # Fire 503s until the error span reaches the collector. The wait + # condition is the same keyed check as the assertion below: a bare + # substring search would also match a trace id or timestamp that + # happens to contain these bytes, ending the poll before the 503 span + # was actually exported. body = b'' found = False for _ in range(int(EXPORT_TIMEOUT * 10)): @@ -224,14 +263,24 @@ def test_otel_span_5xx_status(tmp_path, protocol): with open(dump, 'rb') as f: body = f.read() - if b'503' in body: + if _has_keyed_int_attr(body, STATUS_CODE_KEY, 503): found = True break time.sleep(0.1) - assert found, 'span with status_code=503 was not exported' - assert b'http.response.status_code' in body, 'status_code attr missing' + assert STATUS_CODE_KEY in body, 'status_code attr missing' + assert found, ( + 'http.response.status_code must carry 503 as an int_value varint; ' + 'a span with that attribute was never exported' + ) + + # The readiness spans are still in the same dump and carry int_value + # 200, which confirms the keyed search discriminates by value rather + # than merely finding the key. + assert _has_keyed_int_attr(body, STATUS_CODE_KEY, 200), ( + 'readiness spans should still carry status_code=200' + ) finally: _kill(proc) From 6757ce6bd8311ac5057e29120822cc78e320f6ed Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 21 Aug 2026 23:28:29 +0200 Subject: [PATCH 031/391] port: validate the size of a queue a peer hands over Every port queue that arrives over a message is mapped for sizeof(nxt_port_queue_t) without asking how large the object behind the descriptor is. mmap() succeeds on a shorter object, and the first access past its last page raises SIGBUS in the process that mapped it: main or the prototype for PROCESS_READY, main or the router for NEW_PORT. A compromised application worker gets that with a single forged message and a one-page memfd -- cheaper than exhausting the receiver's descriptor table, which is what the previous commit closed. Validate the object with fstat() before mapping it, and refuse one that cannot hold the queue. Only a short object is refused, not a differing one: the senders do ftruncate() the queue to exactly the mapped size, but fstat() does not have to report that back. A shm object is rounded up to a page on some systems, and the queue is not a whole number of pages, so requiring equality would refuse every legitimate queue there -- and in the router that means an application whose port never finishes being set up. The three queue sites now share one helper. This validates rather than closes the hole. The sender keeps the descriptor and can shrink the object after the check, and it cannot be sealed against that: nxt_shm_open() creates the memfd without MFD_ALLOW_SEALING, and a forged queue simply omits the flag. What the check does buy is that a queue which cannot work is refused where it is handed over, with a log line naming the descriptor, instead of faulting at an arbitrary later access. Closing it needs the senders to seal the object and the receiver to require the seal, with a compatibility window for older libunit. The descriptor's type is deliberately not constrained. A legitimate sender uses memfd_create() on Linux and shm_open() elsewhere, and both report S_IFREG on the platforms Unit builds for, so S_ISREG would accept an ordinary file anyway while risking a false reject on a platform whose shm objects report something else. The size is what makes the mapping safe, not the object's flavour. A refused queue is not a reason to refuse the message. PROCESS_READY leaves the port as it was and logs, exactly as a failed mmap() already did, so that a forged message carrying a bad descriptor cannot keep a process from ever being marked ready. The router's NEW_PORT path needs one thing more than the check, because the check is what makes its failure branch worth reaching. It returned as soon as the mapping failed, and that return walked past the nxt_fd_close() below it, so the refused descriptor stayed open; the dispatcher reclaims nothing a handler leaves behind (nxt_port_read_msg_process() calls port->handler() and returns). Until now that branch was taken only when mmap() itself failed, which needs the router to be out of address space; with the size check a peer picks when it happens, one descriptor of ours per message, which is the exhaustion the check exists to prevent. The descriptor is now closed whichever way the mapping went. Two things that same return causes are deliberately left alone. It also walks past the RPC dispatch at the end of the handler, so a refused queue leaves the pending start and its app_joint reference outstanding until the request times out; and the port stays registered even though a port whose queue was refused can never be reached -- libunit hands a socket message to the application only after a READ_SOCKET marker has arrived through the queue, and a queueless sender never emits one. Neither can be fixed at this call site, and attempts to do so are worse than the defects. The port this handler holds is either one that nxt_port_new_port_handler() has just created or one that was already live and serving -- the already-exists branch returns the existing port with fd[1] still set -- and nothing here distinguishes them. Failing the start or releasing the port therefore does it to a working application port whenever a forged duplicate NEW_PORT names one. Fixing them means making provenance visible across the two handlers, which is its own change; #223 tracks it. A test covers the one thing this now claims. src/test/nxt_router_new_port_test.c hands the handler an application NEW_PORT whose fd[1] is a one-page object and asserts the descriptor is closed and cleared, while fd[0] stays with the port that took it. Writing it needed nxt_router_new_port_handler() to stop being static -- it and nxt_router_port_queue_map() both were, which is why this function had no coverage at all -- so it is declared in src/nxt_router.h now, as nxt_port.h already does for the handlers it exports. The two other queue sites the helper covers do not have this shape. nxt_main_new_port_handler() closes fd[1] whatever the mapping returned, and nxt_port_process_ready_handler() closes through nxt_port_recv_msg_close_fds() on every path out. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KMTKtFcf3YhmBPvTSKTkfg --- auto/sources | 1 + src/nxt_main_process.c | 6 +- src/nxt_port.c | 85 +++++++++- src/nxt_port.h | 1 + src/nxt_router.c | 37 +++-- src/nxt_router.h | 3 + src/test/nxt_port_ready_test.c | 71 +++++++++ src/test/nxt_router_new_port_test.c | 236 ++++++++++++++++++++++++++++ src/test/nxt_tests.c | 3 + src/test/nxt_tests.h | 1 + 10 files changed, 426 insertions(+), 18 deletions(-) create mode 100644 src/test/nxt_router_new_port_test.c diff --git a/auto/sources b/auto/sources index f572efa52..c507bd2a3 100644 --- a/auto/sources +++ b/auto/sources @@ -178,6 +178,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_fail_test.c \ src/test/nxt_port_mmap_range_test.c \ src/test/nxt_port_ready_test.c \ + src/test/nxt_router_new_port_test.c \ src/test/nxt_port_change_file_test.c \ " diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index b6d6832a6..de3095c71 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -425,9 +425,9 @@ nxt_main_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) && port->type == NXT_PROCESS_APP && msg->fd[1] != -1) { - mem = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), - PROT_READ | PROT_WRITE, MAP_SHARED, msg->fd[1], 0); - if (nxt_fast_path(mem != MAP_FAILED)) { + mem = nxt_port_queue_mmap(task, msg->fd[1], sizeof(nxt_port_queue_t)); + + if (nxt_fast_path(mem != NULL)) { port->queue = mem; } diff --git a/src/nxt_port.c b/src/nxt_port.c index 09696f964..49ca104fe 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -19,6 +19,73 @@ static void nxt_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static nxt_atomic_uint_t nxt_port_last_id = 1; +/* + * Map a queue a peer sent over a port. + * + * mmap() succeeds when the descriptor refers to an object shorter than the + * mapping, and the first access past the object's last page raises SIGBUS + * in this process -- main, the prototype or the router, depending on which + * handler took the message. That is cheaper for a hostile peer than + * exhausting the descriptor table, so the size has to be validated before + * the mapping is made. + * + * Only a short object is refused. Both senders do size the queue with + * ftruncate() to exactly the size the receiver maps -- nxt_shm_open() in + * the runtime and nxt_unit_shm_open() in libunit -- but fstat() does not + * have to report that size back: a shm object is rounded up to a page on + * some systems, and the queue is not a whole number of pages, so requiring + * the exact size would refuse every legitimate queue there. A larger + * object is harmless -- only the first size bytes are mapped, and every + * access uses fixed offsets derived from the same type. + * + * The check does not make a hostile peer harmless: the sender keeps the + * descriptor and can shrink the object after the check, and neither + * MAP_POPULATE nor a probe read would close that window -- both only touch + * the pages before the shrink. Sealing the object is not available to the + * receiver, and is Linux-only. What the check does close is the whole + * class of short objects a peer can simply hand over, which is what the + * receiver can decide on its own. + */ + +void * +nxt_port_queue_mmap(nxt_task_t *task, nxt_fd_t fd, size_t size) +{ + void *mem; + struct stat queue_stat; + + if (nxt_slow_path(fstat(fd, &queue_stat) == -1)) { + nxt_log(task, NXT_LOG_WARN, "fstat(%FD) failed %E", fd, nxt_errno); + + return NULL; + } + + /* + * Only a short object is refused, not a differing one: the producers + * size the queue exactly, but fstat() does not have to report that. + * A shm object is rounded up to a page on some systems, and the queue + * is not a whole number of pages, so requiring equality would refuse + * every legitimate queue there. + */ + if (nxt_slow_path(queue_stat.st_size < (off_t) size)) { + nxt_log(task, NXT_LOG_WARN, "port queue on descriptor %FD is %O " + "bytes, less than the %uz it must hold", fd, + queue_stat.st_size, size); + + return NULL; + } + + mem = nxt_mem_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + if (nxt_slow_path(mem == MAP_FAILED)) { + nxt_log(task, NXT_LOG_WARN, "mmap(%FD) failed %E", fd, nxt_errno); + + return NULL; + } + + return mem; +} + + static void nxt_port_mp_cleanup(nxt_task_t *task, void *obj, void *data) { @@ -392,9 +459,9 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "process %PI ready", msg->port_msg.pid); if (msg->fd[0] != -1) { - mem = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), - PROT_READ | PROT_WRITE, MAP_SHARED, msg->fd[0], 0); - if (nxt_fast_path(mem != MAP_FAILED)) { + mem = nxt_port_queue_mmap(task, msg->fd[0], sizeof(nxt_port_queue_t)); + + if (nxt_fast_path(mem != NULL)) { /* * Release what a previous PROCESS_READY installed, so that * replacing the queue does not leak the old mapping and its @@ -421,9 +488,15 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) msg->fd[0] = -1; } else { - nxt_log(task, NXT_LOG_WARN, "process %PI ready: queue mmap " - "failed %E, falling back to the socket", msg->port_msg.pid, - nxt_errno); + /* + * A refused queue leaves the port as it was, as a failed + * mapping always did -- on the socket if it had no queue yet, + * on the one it already has otherwise. Refusing the message + * outright would let a forged PROCESS_READY carrying a bad + * descriptor keep the process from ever being marked ready. + */ + nxt_log(task, NXT_LOG_WARN, "process %PI ready: cannot map the " + "queue, leaving the port as it was", msg->port_msg.pid); } } diff --git a/src/nxt_port.h b/src/nxt_port.h index d859ebcec..471350e75 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -407,6 +407,7 @@ nxt_int_t nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, void nxt_port_change_log_file(nxt_task_t *task, nxt_runtime_t *rt, nxt_uint_t slot, nxt_fd_t fd); void nxt_port_remove_notify_others(nxt_task_t *task, nxt_process_t *process); +void *nxt_port_queue_mmap(nxt_task_t *task, nxt_fd_t fd, size_t size); void nxt_port_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/src/nxt_router.c b/src/nxt_router.c index aa3d322b9..9db6c4298 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -106,8 +106,6 @@ static void nxt_router_greet_controller(nxt_task_t *task, static nxt_int_t nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app); -static void nxt_router_new_port_handler(nxt_task_t *task, - nxt_port_recv_msg_t *msg); static void nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_router_app_restart_handler(nxt_task_t *task, @@ -676,7 +674,7 @@ nxt_request_rpc_data_unlink(nxt_task_t *task, } -static void +void nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_int_t res; @@ -709,12 +707,34 @@ nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } else { if (msg->fd[1] != -1) { res = nxt_router_port_queue_map(task, port, msg->fd[1]); - if (nxt_slow_path(res != NXT_OK)) { - return; - } + + /* + * Closed whether or not the mapping succeeded: the mapping does + * not keep the descriptor, and the dispatcher reclaims nothing a + * handler leaves behind. The size check above is what makes this + * matter -- it lets a peer choose to be refused, so the failure + * return below handed that peer a descriptor of ours per message, + * which is the exhaustion the check exists to prevent. + * + * The return itself is left alone. It also walks past the RPC + * dispatch below, so a refused queue leaves the pending start + * outstanding, and the port stays registered although a port + * whose queue was refused can never be reached. Both are worth + * fixing and neither is fixable here: the port this handler is + * holding may be one nxt_port_new_port_handler() just created or + * one that was already live and serving, and nothing at this call + * site can tell them apart -- so failing the start or releasing + * the port would, for a forged duplicate NEW_PORT, do it to a + * working application port. That needs the two handlers to agree + * on provenance first; see #223. + */ nxt_fd_close(msg->fd[1]); msg->fd[1] = -1; + + if (nxt_slow_path(res != NXT_OK)) { + return; + } } } @@ -2751,9 +2771,8 @@ nxt_router_port_queue_map(nxt_task_t *task, nxt_port_t *port, nxt_fd_t fd) nxt_assert(fd != -1); - mem = nxt_mem_mmap(NULL, sizeof(nxt_port_queue_t), - PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (nxt_slow_path(mem == MAP_FAILED)) { + mem = nxt_port_queue_mmap(task, fd, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(mem == NULL)) { return NXT_ERROR; } diff --git a/src/nxt_router.h b/src/nxt_router.h index e2c5b83bc..8ca5dbbea 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -262,6 +262,9 @@ void nxt_router_access_log_use(nxt_thread_spinlock_t *lock, nxt_router_access_log_t *access_log); void nxt_router_access_log_release(nxt_task_t *task, nxt_thread_spinlock_t *lock, nxt_router_access_log_t *access_log); +/* Not static so the descriptor-ownership test can drive it directly. */ +void nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); + void nxt_router_access_log_reopen_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index 61241ee4f..9d5e9f175 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -177,6 +177,72 @@ nxt_port_ready_test_queue(nxt_thread_t *thr, nxt_task_t *task, return NXT_ERROR; } + +/* + * Hand the handler a descriptor too short for the queue. mmap() succeeds + * on it, so nothing fails until the first access past the object's last + * page, which raises SIGBUS in the process that mapped it -- main or the + * prototype. The handler has to refuse the descriptor and keep the queue + * it already had. + */ +static nxt_int_t +nxt_port_ready_test_short_queue(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_port_t *port) +{ + void *prev_queue; + nxt_fd_t fd, prev_fd; + volatile nxt_port_queue_t *queue; + + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + if (fd == -1) { + nxt_log_alert(thr->log, "port ready test failed to create the queue"); + return NXT_ERROR; + } + + /* + * One page: mmap() of the whole queue over it succeeds, and every byte + * beyond it faults. + */ + if (ftruncate(fd, nxt_pagesize) == -1) { + nxt_log_alert(thr->log, "port ready test failed to size the queue"); + (void) close(fd); + return NXT_ERROR; + } + + prev_fd = port->queue_fd; + prev_queue = port->queue; + + msg->fd[0] = fd; + msg->fd[1] = -1; + + nxt_port_process_ready_handler(task, msg); + + if (port->queue != prev_queue || port->queue_fd != prev_fd) { + nxt_log_alert(thr->log, "port ready test: short queue replaced the " + "installed one"); + return NXT_ERROR; + } + + if (msg->fd[0] != -1 || nxt_port_ready_test_fd_is_open(fd)) { + nxt_log_alert(thr->log, "port ready test: short queue leaked its " + "descriptor"); + return NXT_ERROR; + } + + /* + * Touch the last item of the queue the port holds, the way + * nxt_port_queue_send() would, to prove the mapping the port kept is + * backed all the way to its end. The check above already catches a + * handler that installs the short mapping, and catches it before this + * access, which on such a handler takes SIGBUS and kills the test + * binary -- verified by moving this access ahead of the check. + */ + queue = port->queue; + queue->items[NXT_PORT_QUEUE_SIZE - 1].size = 0; + + return NXT_OK; +} + #endif @@ -297,6 +363,11 @@ nxt_port_ready_test(nxt_thread_t *thr) } ret = nxt_port_ready_test_queue(thr, task, &msg, port, "replacing ready"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_ready_test_short_queue(thr, task, &msg, port); #endif diff --git a/src/test/nxt_router_new_port_test.c b/src/test/nxt_router_new_port_test.c new file mode 100644 index 000000000..62d0f07b2 --- /dev/null +++ b/src/test/nxt_router_new_port_test.c @@ -0,0 +1,236 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership in + * nxt_router_new_port_handler() (src/nxt_router.c). + * + * An application NEW_PORT carries the worker's socket in fd[0] and the + * queue it wants the router to map in fd[1]. The router maps the queue + * and closes fd[1] afterwards -- but the failure branch of that mapping + * used to return before the close, leaking the descriptor. + * + * That branch used to need the router to be out of address space, which is + * why it went unnoticed. Since the queue object is validated for size, a + * peer decides when it is taken: it hands over a short object and the + * router refuses it. The dispatcher reclaims nothing once a handler has + * returned (see nxt_port_recv_msg_close_fds()), so each refusal cost the + * router a descriptor, which is the exhaustion the validation exists to + * prevent. + * + * The port keeps fd[0]: it is the port's socket, and the port outlives the + * message. Only fd[1] is the message's to lose. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + +#if (NXT_HAVE_MEMFD_CREATE) +#include +#include +#endif + + +#if (NXT_HAVE_MEMFD_CREATE) + +static nxt_bool_t +nxt_router_new_port_test_fd_is_open(nxt_fd_t fd) +{ + return fcntl(fd, F_GETFD) != -1; +} + +#endif + + +nxt_int_t +nxt_router_new_port_test(nxt_thread_t *thr) +{ +#if !(NXT_HAVE_MEMFD_CREATE) + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router new port test skipped: no memfd_create()"); + return NXT_OK; + +#else + + nxt_mp_t *mp; + nxt_fd_t sock, queue; + nxt_buf_t buf; + nxt_task_t *task; + nxt_int_t ret; + nxt_port_t *port; + nxt_runtime_t *rt, *saved_rt; + nxt_event_engine_t engine; + nxt_port_recv_msg_t msg; + nxt_port_msg_new_port_t new_port; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router new port test started"); + + ret = NXT_ERROR; + sock = -1; + queue = -1; + port = NULL; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * The minimal engine nxt_port_write_enable() needs, as + * nxt_port_fail_test.c injects one: only fast_work_queue is set up, so + * a path that reaches another member would read garbage. The handler + * under test does not get that far. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + saved_rt = thr->runtime; + thr->runtime = rt; + thr->engine = &engine; + + /* + * Releasing the port's last reference reaches nxt_runtime_process_release(), + * which frees inline when task->thread->engine is rt->main_engine and + * otherwise posts the free to that engine -- NULL here, and a posted work + * item would outlive this stack frame anyway. Pointing it at the fixture's + * own engine keeps the teardown synchronous. + */ + rt->main_engine = &engine; + + /* + * A short queue: mmap() of the whole object over it would succeed and + * fault later, so the size check has to refuse it here. + */ + queue = syscall(SYS_memfd_create, "nxt_router_new_port_test", MFD_CLOEXEC); + sock = open("/dev/null", O_RDONLY); + + if (queue == -1 || sock == -1) { + nxt_log_alert(thr->log, "router new port test failed to open the " + "descriptors"); + goto done; + } + + if (ftruncate(queue, nxt_pagesize) == -1) { + nxt_log_alert(thr->log, "router new port test failed to size the " + "queue"); + goto done; + } + + nxt_memzero(&new_port, sizeof(nxt_port_msg_new_port_t)); + + new_port.pid = nxt_pid + 1; + new_port.id = 1; + new_port.type = NXT_PROCESS_APP; + new_port.max_size = 1024; + new_port.max_share = 1024; + + nxt_memzero(&buf, sizeof(nxt_buf_t)); + + buf.mem.pos = (u_char *) &new_port; + buf.mem.free = buf.mem.pos + sizeof(nxt_port_msg_new_port_t); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = &buf; + msg.fd[0] = sock; + msg.fd[1] = queue; + + /* stream 0: the handler returns after the refusal, without RPC. */ + msg.port_msg.stream = 0; + msg.port_msg.pid = new_port.pid; + + nxt_router_new_port_handler(task, &msg); + + if (msg.fd[1] != -1 || nxt_router_new_port_test_fd_is_open(queue)) { + nxt_log_alert(thr->log, "router new port test: a refused queue " + "leaked its descriptor"); + goto done; + } + + queue = -1; + + if (msg.fd[0] != -1) { + nxt_log_alert(thr->log, "router new port test: the port did not take " + "the socket"); + goto done; + } + + /* + * The port owns the socket now, so the test must not close it directly + * -- but the port itself is the fixture's to release: it is registered + * in rt->ports and holds both the descriptor and its own pool, and + * destroying only the runtime pool below would leak them for the rest + * of the test binary. + */ + sock = -1; + port = msg.u.new_port; + + ret = NXT_OK; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router new port test passed"); + +done: + + if (queue != -1) { + (void) close(queue); + } + + if (sock != -1) { + (void) close(sock); + } + + if (port != NULL) { + /* + * Through the ownership path, not by destroying the pool directly: + * nxt_port_mp_cleanup() asserts that the descriptors are gone and + * the use count is zero, so a direct nxt_mp_destroy() aborts a debug + * build even though it looks clean in a release one. close() clears + * pair[1]; remove() drops the rt->ports reference, which is the last + * one, and the port is freed there. + */ + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + } + + thr->runtime = saved_rt; + thr->engine = NULL; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; + +#endif +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index e50da37f6..09ba76972 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -197,6 +197,9 @@ main(int argc, char **argv) if (nxt_port_ready_test(thr) != NXT_OK) { return 1; } + if (nxt_router_new_port_test(thr) != NXT_OK) { + return 1; + } if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index bc2061e52..8332fa8a2 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -73,6 +73,7 @@ nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); +nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From cc9b3e99392f2a77a83a33c9a64029ff0a2fef41 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 17:51:46 +0200 Subject: [PATCH 032/391] ci(build-test): refresh the apt index before installing libperl-dev The perl leg fails on every PR in "Install libperl-dev": E: Failed to fetch mirror+file:/etc/apt/apt-mirrors.txt/pool/main/p/perl/ libperl-dev_5.38.2-3.2ubuntu0.3_amd64.deb 404 Not Found E: Unable to fetch some archives, maybe run apt-get update [...] (run 33183190831, job 98894408092, exit 100; attempt 2 failed identically, so it is not a transient mirror hiccup.) The runner image ships a package index that has gone stale: Ubuntu has since published a newer libperl-dev and dropped the pinned .deb from the pool, so the cached index points at a URL that no longer exists. The step installed without a preceding `apt-get update`, so apt had no way to learn the new version. Three other steps in this workflow had the same gap -- "Install packages", "Build OpenSSL 3.6" and "Install pytest" -- and are equally exposed the next time one of their packages is superseded, so refresh the index in all four. While here, make the two non-`-y` invocations non-interactive to match the rest of the file. Co-Authored-By: Claude Opus --- .github/workflows/build-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index a147c4d13..701234107 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -308,7 +308,8 @@ jobs: - name: Install libperl-dev run: | - sudo apt-get install libperl-dev + sudo apt-get -y update + sudo apt-get -y install libperl-dev if: steps.metadata.outputs.module == 'perl' - name: Configure perl From 6f937727f2fd65b93f237ee28b308eeec6ed48b0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 22 Aug 2026 02:28:28 +0200 Subject: [PATCH 033/391] port: say when NEW_PORT created no port nxt_port_new_port_handler() assigned msg->u.new_port only on the paths that end with a port, and returned without touching it when the port already existed or nxt_runtime_process_port_create() failed. The message lives on the stack of nxt_port_read_handler() and nothing clears it, so on those paths the union held whatever the stack happened to contain. Every wrapper reads it back. nxt_controller_process_new_port_handler() dereferenced it unconditionally, so a NEW_PORT that creates no port made the controller follow a pointer built from stack garbage; main and the router check for NULL, which does not help against a non-NULL value that was never a port. Set it to NULL on entry and have the controller check it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KMTKtFcf3YhmBPvTSKTkfg --- src/nxt_controller.c | 7 ++++++- src/nxt_port.c | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/nxt_controller.c b/src/nxt_controller.c index b46d19570..d51f4154a 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -411,9 +411,14 @@ static void nxt_controller_process_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { + nxt_port_t *port; + nxt_port_new_port_handler(task, msg); - if (msg->u.new_port->type != NXT_PROCESS_ROUTER + port = msg->u.new_port; + + if (port == NULL + || port->type != NXT_PROCESS_ROUTER || !nxt_controller_router_ready) { return; diff --git a/src/nxt_port.c b/src/nxt_port.c index 49ca104fe..0f27f65aa 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -350,6 +350,14 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; + /* + * The message arrives on the stack of nxt_port_read_handler() with the + * union uninitialized, and every caller of this handler reads + * msg->u.new_port on return, so a path that creates no port has to say + * so rather than leave the garbage the stack happened to hold. + */ + msg->u.new_port = NULL; + new_port_msg = (nxt_port_msg_new_port_t *) msg->buf->mem.pos; /* TODO check b size and make plain */ From 73cc8cc857b1293eea51fd3dbfc68fded8400305 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 22 Aug 2026 02:28:43 +0200 Subject: [PATCH 034/391] port: close the descriptors a NEW_PORT or MMAP message leaves behind A port message carries up to two descriptors whatever its type normally uses, and the dispatcher does not reclaim them once the handler returns, so every path that does not consume one has to close it. Several paths around NEW_PORT and MMAP did not. nxt_port_mmap_handler() never closed the second descriptor on any path, and returned closing nothing at all when the first was missing. nxt_port_new_port_handler() closed only the first when the port already existed, and neither when the port could not be created. The wrappers were no better: main kept the second descriptor for every port that was not a newly created application port, the router lost it when the queue could not be mapped and on the branches that map no queue at all, the controller never touched it, and the discovery and prototype processes wired the bare handler, which left it open for the life of the process. The descriptor now has one owner at each point: the base handler leaves it set only where a wrapper still has something to do with it, and closes it everywhere else. Each wrapper closes what it is handed once it is done, and the discovery and prototype processes get a wrapper of their own that simply closes it. Both handlers also close what a message brought when they refuse it, as does the dispatcher itself for a message type it has no handler for -- the type is a byte off the wire, so any peer can name one. On the path where the port already exists, the queue descriptor is refused only when the port already has a queue. Mapping it over a live queue installs a second mapping and leaks the one it replaces, which would let a repeated NEW_PORT re-point an established port at memory of the sender's choosing. An existing port without a queue is a different case, and it is main's normal path rather than an anomaly: nxt_main_process_whoami_handler() creates every application port when the worker reports in, before the prototype's PROCESS_READY makes main forward that port's queue. So this branch is the only place main is ever handed a worker's queue. Refusing it there left port->queue NULL for every worker, and a queueless sender falls back to a plain socket write that libunit parks waiting for a READ_SOCKET marker that never comes -- stranding every CHANGE_FILE and QUIT main sends, and wedging a worker on the first log rotation. Guard the dispatcher against an in-range type whose slot is unset as well. The handler tables are sparse designated initializers, so most slots in most of them are NULL, and a peer naming an unfilled type called a NULL pointer rather than reaching a handler. The queue-map failure branch is left alone here. It leaks the descriptor it refused and strands the start RPC, and both are real, but neither can be answered at that call site: nxt_port_new_port_handler() returns either a port it has just created or one that was already live and serving -- its already-exists branch hands back the existing port -- and nothing there tells them apart, so failing the start or releasing the port does it to a working application port whenever a forged duplicate NEW_PORT names one. The descriptor close belongs to #211, which is where the size check makes that branch reachable at all; #223 tracks making the provenance visible so the rest can be fixed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PJBDxzs854PVkCVTF7XB5K --- auto/sources | 1 + src/nxt_application.c | 23 +- src/nxt_controller.c | 6 + src/nxt_main_process.c | 12 +- src/nxt_port.c | 83 ++++++- src/nxt_router.c | 11 + src/test/nxt_port_fd_test.c | 477 ++++++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 9 files changed, 603 insertions(+), 15 deletions(-) create mode 100644 src/test/nxt_port_fd_test.c diff --git a/auto/sources b/auto/sources index c507bd2a3..75ef9930d 100644 --- a/auto/sources +++ b/auto/sources @@ -180,6 +180,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_ready_test.c \ src/test/nxt_router_new_port_test.c \ src/test/nxt_port_change_file_test.c \ + src/test/nxt_port_fd_test.c \ " diff --git a/src/nxt_application.c b/src/nxt_application.c index 9f476c996..af61dbc50 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -67,6 +67,8 @@ static u_char *nxt_cstr_dup(nxt_mp_t *mp, u_char *dst, u_char *src); static void nxt_proto_signal_handler(nxt_task_t *task, void *obj, void *data); static void nxt_proto_sigterm_handler(nxt_task_t *task, void *obj, void *data); static void nxt_proto_sigchld_handler(nxt_task_t *task, void *obj, void *data); +static void nxt_app_new_port_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); nxt_str_t nxt_server = nxt_string(NXT_SERVER); @@ -87,7 +89,7 @@ static nxt_common_app_conf_t *nxt_app_conf; static const nxt_port_handlers_t nxt_discovery_process_port_handlers = { .quit = nxt_signal_quit_handler, - .new_port = nxt_port_new_port_handler, + .new_port = nxt_app_new_port_handler, .change_file = nxt_port_change_log_file_handler, .mmap = nxt_port_mmap_handler, .data = nxt_port_data_handler, @@ -110,7 +112,7 @@ const nxt_sig_event_t nxt_prototype_signals[] = { static const nxt_port_handlers_t nxt_proto_process_port_handlers = { .quit = nxt_proto_quit_handler, .change_file = nxt_port_change_log_file_handler, - .new_port = nxt_port_new_port_handler, + .new_port = nxt_app_new_port_handler, .process_created = nxt_proto_process_created_handler, .process_ready = nxt_port_process_ready_handler, .remove_pid = nxt_port_remove_pid_handler, @@ -161,6 +163,23 @@ const nxt_process_init_t nxt_app_process = { }; +/* + * nxt_port_new_port_handler() hands the queue descriptor of a newly created + * port to its caller, because only the caller knows whether this process + * maps port queues. Neither the discovery nor the prototype process does, + * so both used the base handler directly and kept every queue descriptor + * their peer sent open for the life of the process. + */ + +static void +nxt_app_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_new_port_handler(task, msg); + + nxt_port_recv_msg_close_fds(msg); +} + + static nxt_int_t nxt_discovery_start(nxt_task_t *task, nxt_process_data_t *data) { diff --git a/src/nxt_controller.c b/src/nxt_controller.c index d51f4154a..2f398cc9d 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -417,6 +417,12 @@ nxt_controller_process_new_port_handler(nxt_task_t *task, port = msg->u.new_port; + /* + * The controller never maps a port queue, so the descriptor + * nxt_port_new_port_handler() leaves to its caller has no use here. + */ + nxt_port_recv_msg_close_fds(msg); + if (port == NULL || port->type != NXT_PROCESS_ROUTER || !nxt_controller_router_ready) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index de3095c71..ad2cec617 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -430,10 +430,16 @@ nxt_main_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_fast_path(mem != NULL)) { port->queue = mem; } - - nxt_fd_close(msg->fd[1]); - msg->fd[1] = -1; } + + /* + * nxt_port_new_port_handler() leaves the queue descriptor to its caller, + * and only a new application port has a use for it here. Anything else + * -- a port that already existed, a port that could not be created, a + * type whose queue main never maps -- used to keep the descriptor open + * in the most privileged process of all. + */ + nxt_port_recv_msg_close_fds(msg); } diff --git a/src/nxt_port.c b/src/nxt_port.c index 0f27f65aa..c4653f99a 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -257,22 +257,37 @@ nxt_port_enable(nxt_task_t *task, nxt_port_t *port, static void nxt_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - nxt_port_handler_t *handlers; + nxt_port_handler_t handler, *handlers; if (nxt_fast_path(msg->port_msg.type < NXT_PORT_MSG_MAX)) { - nxt_debug(task, "port %d: message type:%uD fds:%d,%d", - msg->port->socket.fd, msg->port_msg.type, - msg->fd[0], msg->fd[1]); - handlers = msg->port->data; - handlers[msg->port_msg.type](task, msg); + handler = handlers[msg->port_msg.type]; - return; + /* + * The tables are designated initializers over a struct of named + * slots, and most of them set only a few: an in-range type whose + * slot the receiving process never filled is NULL, not a handler. + */ + if (nxt_fast_path(handler != NULL)) { + nxt_debug(task, "port %d: message type:%uD fds:%d,%d", + msg->port->socket.fd, msg->port_msg.type, + msg->fd[0], msg->fd[1]); + + handler(task, msg); + + return; + } } nxt_alert(task, "port %d: unknown message type:%uD", msg->port->socket.fd, msg->port_msg.type); + + /* + * The type is a byte off the wire, so any peer can name one that has + * no handler. Nothing runs to take the descriptors it attached. + */ + nxt_port_recv_msg_close_fds(msg); } @@ -372,8 +387,36 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) msg->u.new_port = port; - nxt_fd_close(msg->fd[0]); - msg->fd[0] = -1; + /* The socket is refused: the port keeps the pair it was built with. */ + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; + } + + /* + * The queue descriptor is left to the caller unless the port has a + * queue already. An existing port without one is not an anomaly + * but the normal path in the main process: + * nxt_main_process_whoami_handler() creates every application port + * at WHOAMI time, before the NEW_PORT + * announcing that same port -- and its queue -- arrives, so this + * branch is the only place main can ever be handed the queue of a + * worker. Refusing fd[1] here outright leaves port->queue NULL, and + * a queueless sender falls back to a plain socket write that libunit + * parks waiting for a READ_SOCKET marker which never comes, stranding + * every CHANGE_FILE and QUIT main sends to its workers. + * + * A port that already has a queue does refuse the descriptor: mapping + * it again would re-point a live port's queue at memory this message + * chose, dropping the mapping the port was built with. Every caller + * maps fd[1] only while it is still set, so clearing it here + * suppresses that without racing them. + */ + if (port->queue != NULL && msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); + msg->fd[1] = -1; + } + return; } @@ -381,6 +424,7 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) new_port_msg->id, new_port_msg->type); if (nxt_slow_path(port == NULL)) { + nxt_port_recv_msg_close_fds(msg); return; } @@ -400,6 +444,15 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_write_enable(task, port); msg->u.new_port = port; + + /* + * fd[1] is deliberately left in the message: it is the queue of the + * port just created, and only the caller knows whether this process + * maps port queues at all and with which size. The same applies on + * the branch above when the port exists but has no queue yet. Every + * caller therefore has to close it once it is done -- which is why the + * refusing paths clear it instead, so that "still set" means "yours". + */ } /* TODO move to nxt_main_process.c */ @@ -529,6 +582,11 @@ nxt_port_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_slow_path(msg->fd[0] == -1)) { nxt_log(task, NXT_LOG_WARN, "invalid fd passed with mmap message"); + /* + * A message can carry two descriptors whichever its type needs, so + * a missing fd[0] does not mean the message carried nothing. + */ + nxt_port_recv_msg_close_fds(msg); return; } @@ -559,7 +617,12 @@ nxt_port_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) fail_close: - nxt_fd_close(msg->fd[0]); + /* + * nxt_port_incoming_port_mmap() maps the segment rather than keeping the + * descriptor, so fd[0] is released on the success path too. fd[1] is + * never used by this handler and was leaked on every one of them. + */ + nxt_port_recv_msg_close_fds(msg); } diff --git a/src/nxt_router.c b/src/nxt_router.c index 9db6c4298..56e265be4 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -690,13 +690,24 @@ nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_router_greet_controller(task, msg->u.new_port); } + /* + * The router maps a queue for application ports only, so every other + * branch below has to release the queue descriptor that + * nxt_port_new_port_handler() left behind. None of the RPC handlers + * reachable from here reads msg->fd, so closing before dispatching them + * takes nothing away. + */ + if (port != NULL && port->type == NXT_PROCESS_PROTOTYPE) { + nxt_port_recv_msg_close_fds(msg); + nxt_port_rpc_handler(task, msg); return; } if (port == NULL || port->type != NXT_PROCESS_APP) { + nxt_port_recv_msg_close_fds(msg); if (msg->port_msg.stream == 0) { return; diff --git a/src/test/nxt_port_fd_test.c b/src/test/nxt_port_fd_test.c new file mode 100644 index 000000000..5e93fc1e1 --- /dev/null +++ b/src/test/nxt_port_fd_test.c @@ -0,0 +1,477 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership in nxt_port_new_port_handler() + * and nxt_port_mmap_handler() (src/nxt_port.c). + * + * A port message can carry two descriptors whatever its type normally uses + * -- nxt_socket_msg_oob_get() accepts an SCM_RIGHTS payload of one or two -- + * and the dispatcher does not reclaim them once the handler returns. Every + * handler therefore has to release what it does not consume, and clear the + * slot of what it does. Both handlers here ignored fd[1] entirely, and + * nxt_port_mmap_handler() returned from its "invalid fd" guard without + * closing anything at all, so a peer attaching descriptors to forged + * messages could exhaust the descriptor table of the router or, through the + * NEW_PORT path, of the main process. + * + * This lives beside nxt_port_ready_test.c rather than inside it: that test + * builds a fixture for a single handler and walks its reject paths, while + * these two handlers need a port hash and a message body instead. The + * dispatcher (nxt_port_handler()) is covered here too: a type past the + * table, and an in-range type whose slot the handler table never filled, + * both have to close what the message carried without calling anything. + * + * The one path this cannot reach is nxt_runtime_process_port_create() + * returning NULL, which needs an allocation failure the test harness has no + * way to inject. + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + + +static nxt_uint_t nxt_port_fd_test_dispatched; + + +static void +nxt_port_fd_test_enable_read_stub(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + /* The fixture engine polls nothing. */ +} + + +/* + * A handler that honours the dispatch contract: consume or close what the + * message carries. Used to pin that the NULL-slot guard does not swallow + * types whose slot is set. + */ +static void +nxt_port_fd_test_dispatch_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_fd_test_dispatched++; + + nxt_port_recv_msg_close_fds(msg); +} + + +static const nxt_port_handlers_t nxt_port_fd_test_handlers = { + .quit = nxt_port_fd_test_dispatch_handler, +}; + + +static nxt_bool_t +nxt_port_fd_test_is_open(nxt_fd_t fd) +{ + return fcntl(fd, F_GETFD) != -1; +} + + +static void +nxt_port_fd_test_close(nxt_fd_t fd) +{ + if (fd != -1 && nxt_port_fd_test_is_open(fd)) { + (void) close(fd); + } +} + + +/* + * Attach two descriptors to the message, run the handler, and expect it to + * have released both: neither of these handlers keeps a descriptor of its + * own beyond the call. + */ +static nxt_int_t +nxt_port_fd_test_closed(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, + void (*handler)(nxt_task_t *task, nxt_port_recv_msg_t *msg), + nxt_bool_t first_fd, const char *name) +{ + nxt_fd_t fd0, fd1; + + fd0 = first_fd ? open("/dev/null", O_RDONLY) : -1; + fd1 = open("/dev/null", O_RDONLY); + + if ((first_fd && fd0 == -1) || fd1 == -1) { + nxt_log_alert(thr->log, "port fd test failed to open /dev/null"); + goto fail; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + handler(task, msg); + + if ((first_fd && nxt_port_fd_test_is_open(fd0)) + || nxt_port_fd_test_is_open(fd1)) + { + nxt_log_alert(thr->log, "port fd test: %s leaked a descriptor", name); + goto fail; /* closes only what is still open */ + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port fd test: %s left fds in the message", + name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + /* + * Close only what the handler left open: a descriptor it consumed is + * gone already, and closing the number again could reach an unrelated + * descriptor that has reused it. + */ + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_port_fd_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_buf_t *b; + nxt_fd_t fd0, fd1; + nxt_task_t *task; + nxt_int_t ret; + nxt_port_t *port, *existing; + nxt_runtime_t *rt, *saved_rt; + nxt_port_recv_msg_t msg; + nxt_event_engine_t *engine, *saved_engine; + nxt_port_msg_new_port_t new_port_msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port fd test started"); + + task = thr->task; + task->thread = thr; + + /* Defined before the first goto done: the cleanup there releases them. */ + port = NULL; + existing = NULL; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + engine = nxt_mp_zalloc(mp, sizeof(nxt_event_engine_t)); + b = nxt_mp_zalloc(mp, sizeof(nxt_buf_t)); + + if (nxt_slow_path(rt == NULL || engine == NULL || b == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + thr->runtime = rt; + + /* + * nxt_port_write_enable() takes the address of a work queue inside the + * engine, so a zeroed one is enough to reach the end of the handler. + */ + thr->engine = engine; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + /* + * The mmap handler refuses a message without a segment descriptor. The + * refusal used to return without closing the second descriptor, which + * the sender is free to attach regardless of the message type. + */ + ret = nxt_port_fd_test_closed(thr, task, &msg, nxt_port_mmap_handler, 0, + "mmap without a segment fd"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A segment announced by a process the runtime does not know: the + * handler releases the segment descriptor on that path but used to keep + * the second one. + */ + msg.port_msg.pid = nxt_pid + 1; + + ret = nxt_port_fd_test_closed(thr, task, &msg, nxt_port_mmap_handler, 1, + "mmap from an unknown process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * NEW_PORT naming a port that exists but has no queue yet. This is + * the normal path in the main process -- WHOAMI creates every + * application port before its NEW_PORT arrives -- so the socket is + * refused, but the queue descriptor has to reach the caller: closing + * it here would leave main's worker ports queueless and strand every + * CHANGE_FILE and QUIT main sends through them. + */ + nxt_memzero(&new_port_msg, sizeof(nxt_port_msg_new_port_t)); + + new_port_msg.pid = nxt_pid + 2; + new_port_msg.id = 7; + new_port_msg.type = NXT_PROCESS_APP; + + b->mem.pos = (u_char *) &new_port_msg; + msg.buf = b; + + existing = nxt_runtime_process_port_create(task, rt, new_port_msg.pid, + new_port_msg.id, + new_port_msg.type); + if (nxt_slow_path(existing == NULL)) { + ret = NXT_ERROR; + goto done; + } + + existing->pair[0] = -1; + existing->pair[1] = -1; + existing->socket.fd = -1; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "port fd test failed to open /dev/null"); + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + msg.fd[0] = fd0; + msg.fd[1] = fd1; + msg.u.new_port = NULL; + + nxt_port_new_port_handler(task, &msg); + + if (nxt_slow_path(msg.u.new_port != existing)) { + nxt_log_alert(thr->log, "port fd test: an existing port was not " + "reported back to the caller"); + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + if (nxt_slow_path(msg.fd[0] != -1 || nxt_port_fd_test_is_open(fd0))) { + nxt_log_alert(thr->log, "port fd test: an existing port did not " + "refuse the socket"); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + if (nxt_slow_path(msg.fd[1] != fd1 || !nxt_port_fd_test_is_open(fd1))) { + nxt_log_alert(thr->log, "port fd test: the queue descriptor of a " + "queueless existing port was not left to the caller"); + ret = NXT_ERROR; + goto done; + } + + nxt_port_fd_test_close(fd1); + msg.fd[1] = -1; + + /* + * NEW_PORT naming a port whose queue is already installed. Here the + * queue descriptor is refused with the socket: a caller that mapped + * fd[1] would re-point a live port's queue at memory this message + * chose. The mapping is a real one so that the cleanup below releases + * it the same way nxt_port_close() releases any port queue. + */ + existing->queue = mmap(NULL, sizeof(nxt_port_queue_t), + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + if (nxt_slow_path(existing->queue == MAP_FAILED)) { + existing->queue = NULL; + nxt_log_alert(thr->log, "port fd test failed to map a queue"); + ret = NXT_ERROR; + goto done; + } + + msg.u.new_port = NULL; + + ret = nxt_port_fd_test_closed(thr, task, &msg, nxt_port_new_port_handler, + 1, "new port that already has a queue"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(msg.u.new_port != existing)) { + nxt_log_alert(thr->log, "port fd test: an existing port was not " + "reported back to the caller"); + ret = NXT_ERROR; + goto done; + } + + /* + * The one path that keeps a descriptor: a port is created, it adopts + * the socket, and the queue descriptor is deliberately left in the + * message for the caller to map or discard. Pinned here because every + * caller now treats "fd[1] still set" as "this one is mine". + */ + new_port_msg.id = 8; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "port fd test failed to open /dev/null"); + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + msg.fd[0] = fd0; + msg.fd[1] = fd1; + msg.u.new_port = NULL; + + nxt_port_new_port_handler(task, &msg); + + port = msg.u.new_port; + + if (nxt_slow_path(port == NULL || port == existing)) { + nxt_log_alert(thr->log, "port fd test: a new port was not created"); + nxt_port_fd_test_close(fd0); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + if (nxt_slow_path(msg.fd[0] != -1 || port->pair[1] != fd0)) { + nxt_log_alert(thr->log, "port fd test: the new port did not take the " + "socket over"); + nxt_port_fd_test_close(fd1); + ret = NXT_ERROR; + goto done; + } + + if (nxt_slow_path(msg.fd[1] != fd1 || !nxt_port_fd_test_is_open(fd1))) { + nxt_log_alert(thr->log, "port fd test: the queue descriptor was not " + "left to the caller"); + ret = NXT_ERROR; + goto done; + } + + nxt_port_fd_test_close(fd1); + msg.fd[1] = -1; + + /* + * The dispatcher itself. The message type is a byte off the wire, so a + * peer can name one past the table -- or, just as easily, an in-range + * one whose slot this process never filled: the tables are designated + * initializers over a struct of named members, and most set only a few. + * Neither may call anything, and both have to close what the message + * carried, since no handler runs to take the descriptors. + * + * nxt_port_enable() installs the (static) dispatcher on the port; the + * fixture engine gets a no-op enable_read so that the read side of the + * enabling touches nothing real. + */ + engine->event.enable_read = nxt_port_fd_test_enable_read_stub; + + nxt_port_enable(task, port, &nxt_port_fd_test_handlers); + + msg.port = port; + + msg.port_msg.type = NXT_PORT_MSG_MAX; + + ret = nxt_port_fd_test_closed(thr, task, &msg, port->handler, 1, + "message type past the handler table"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + msg.port_msg.type = _NXT_PORT_MSG_NEW_PORT; /* slot not set above */ + + ret = nxt_port_fd_test_closed(thr, task, &msg, port->handler, 1, + "message type with an empty handler slot"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(nxt_port_fd_test_dispatched != 0)) { + nxt_log_alert(thr->log, "port fd test: a refused type reached a " + "handler"); + ret = NXT_ERROR; + goto done; + } + + /* And a slot that is set still dispatches. */ + msg.port_msg.type = _NXT_PORT_MSG_QUIT; + + ret = nxt_port_fd_test_closed(thr, task, &msg, port->handler, 1, + "message type with its slot set"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(nxt_port_fd_test_dispatched != 1)) { + nxt_log_alert(thr->log, "port fd test: a handled type did not reach " + "its handler"); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_OK; + +done: + + /* + * The ports hold what their handlers adopted -- the new port its socket + * descriptor, the existing one its queue mapping -- and nothing else + * owns them here: leaving them to the pool teardown would leak both. + * + * close() releases those, but not the port: each one carries its own + * memory pool and a reference held by rt->ports, neither of which the + * nxt_mp_destroy() below can reach. remove() drops that reference, + * which is the last one, and frees the port -- so it has to happen + * while the fixture runtime is still installed. + */ + if (existing != NULL) { + nxt_port_close(task, existing); + nxt_runtime_port_remove(task, existing); + } + + if (port != NULL) { + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + } + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port fd test passed"); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 09ba76972..7b256d8b7 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -205,6 +205,10 @@ main(int argc, char **argv) return 1; } + if (nxt_port_fd_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CGROUP) if (nxt_cgroup_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 8332fa8a2..7206e1037 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -75,6 +75,7 @@ nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); +nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From 1a9f31b6ff68bbccc7e44b0fda51200ad319b29a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 18:18:03 +0200 Subject: [PATCH 035/391] test(port): make the queue-size tests able to fail The router-side test added with the size check asserted only descriptor ownership, and that holds with or without the check it exists to guard: without it, nxt_router_port_queue_map() maps sizeof(nxt_port_queue_t) over a short object and succeeds -- that is the defect, the fault arrives later -- and on that path the descriptor is closed and fd[0] is taken just the same. Assert instead that the refused queue was never installed: port->queue stays NULL, which is what "refused" means. Proving that assertion can fail needed the fixture to change. With stream 0 the reverted handler runs past the refusal into nxt_assert(main_app_port != NULL), and satisfying that only moves the failure to the PORT_ACK write, which enqueues onto the short mapping and faults -- the run dies of the bug under test rather than of the assertion. The test now hands the handler a real reply port and stream 1, so the reverted path leaves through the RPC dispatch before either hazard. nxt_port_new() zeroes the port, so pair[0], pair[1] and socket.fd are set to -1 explicitly: otherwise teardown closes fd 0. Two boundary cases cover the rule the helper actually implements, as direct calls to nxt_port_queue_mmap(). An object of exactly sizeof(nxt_port_queue_t) is accepted, which pins the comparison as "<" rather than "<=". A page-rounded object is accepted too, which pins the deliberate decision not to require equality: sizeof(nxt_port_queue_t) is 160 pages plus 20 bytes, so a shm object rounded up to a page reports more than the queue needs, and "tightening" the check to == would refuse every legitimate queue on such a platform and stall application port setup. Each of the three was shown to fail on the revert it guards, not argued to: "<" to "<=" fails the exact-size case, "<" to "!=" fails the page-rounded case, and removing the size check fails the router case with a clean message and no fault. The last of those is only visible with the ready test's short-queue case gated -- ungated it fails first and the router test never starts, which is the false-positive this was written to remove and which does bite. Closes #228. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013uPosiJqof76yNnasGdgFn --- src/test/nxt_port_ready_test.c | 97 +++++++++++++++++++++++++++++ src/test/nxt_router_new_port_test.c | 56 ++++++++++++++++- 2 files changed, 150 insertions(+), 3 deletions(-) diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index 9d5e9f175..f60fe3f05 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -98,6 +98,89 @@ nxt_port_ready_test_case(nxt_thread_t *thr, nxt_task_t *task, #if (NXT_HAVE_MEMFD_CREATE) +/* + * The size check in nxt_port_queue_mmap() (src/nxt_port.c) refuses only an + * object shorter than the queue, and both edges of that rule are load + * bearing. + * + * An object of exactly sizeof(nxt_port_queue_t) is what both producers ask + * for -- nxt_shm_open() in the runtime and nxt_unit_shm_open() in libunit + * -- so the comparison has to be "<": a "<=" would refuse every queue. + * + * A longer object is accepted on purpose. A shm object is rounded up to a + * page on some systems and the queue is not a whole number of pages, so + * tightening the check to equality would refuse every legitimate queue + * there and stall application port setup. Only the first + * sizeof(nxt_port_queue_t) bytes are mapped either way. + * + * Both cases touch the last item of the mapping the way + * nxt_port_queue_send() would. That probe is not a SIGBUS check: every page + * of an accepted mapping is at least partially file-backed here, and a + * partial page at EOF is zero-filled, so no size the helper accepts can + * fault there. What it catches is the helper handing back a short or + * garbage mapping -- and it fixes the shape of the access, so that a future + * change to what gets mapped is exercised rather than merely returned. The + * teeth of these two cases are the NULL checks above it. + */ +static nxt_int_t +nxt_port_ready_test_queue_size(nxt_thread_t *thr, nxt_task_t *task, + size_t file_size, const char *name) +{ + void *mem; + nxt_fd_t fd; + volatile nxt_port_queue_t *queue; + + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + if (fd == -1) { + nxt_log_alert(thr->log, "port ready test failed to create the queue"); + return NXT_ERROR; + } + + if (ftruncate(fd, file_size) == -1) { + nxt_log_alert(thr->log, "port ready test failed to size the queue"); + (void) close(fd); + return NXT_ERROR; + } + + mem = nxt_port_queue_mmap(task, fd, sizeof(nxt_port_queue_t)); + + /* The mapping does not keep the descriptor. */ + (void) close(fd); + + if (mem == NULL) { + nxt_log_alert(thr->log, "port ready test: %s queue was refused", name); + return NXT_ERROR; + } + + queue = mem; + queue->items[NXT_PORT_QUEUE_SIZE - 1].size = 0; + + nxt_mem_munmap(mem, sizeof(nxt_port_queue_t)); + + return NXT_OK; +} + + +static nxt_int_t +nxt_port_ready_test_queue_sizes(nxt_thread_t *thr, nxt_task_t *task) +{ + size_t rounded; + + if (nxt_slow_path(nxt_port_ready_test_queue_size(thr, task, + sizeof(nxt_port_queue_t), + "an exact-size") + != NXT_OK)) + { + return NXT_ERROR; + } + + rounded = nxt_align_size(sizeof(nxt_port_queue_t), nxt_pagesize); + + return nxt_port_ready_test_queue_size(thr, task, rounded, + "a page-rounded"); +} + + /* * Hand the handler a mappable descriptor and check that the port takes it * over: the queue is installed, whatever the port held before is released, @@ -263,6 +346,20 @@ nxt_port_ready_test(nxt_thread_t *thr) task = thr->task; task->thread = thr; +#if (NXT_HAVE_MEMFD_CREATE) + + /* + * The two accepted sizes, ahead of the handler fixtures and calling + * nxt_port_queue_mmap() directly: nothing is allocated yet, and a + * failure here names the rule that broke rather than the handler that + * relied on it. + */ + if (nxt_slow_path(nxt_port_ready_test_queue_sizes(thr, task) != NXT_OK)) { + return NXT_ERROR; + } + +#endif + /* Defined before the first goto done: the cleanup there releases it. */ port = NULL; diff --git a/src/test/nxt_router_new_port_test.c b/src/test/nxt_router_new_port_test.c index 62d0f07b2..7af5644bf 100644 --- a/src/test/nxt_router_new_port_test.c +++ b/src/test/nxt_router_new_port_test.c @@ -21,6 +21,12 @@ * * The port keeps fd[0]: it is the port's socket, and the port outlives the * message. Only fd[1] is the message's to lose. + * + * The refusal itself is checked here too: the port must come back without a + * queue. A descriptor accounted for proves nothing about the mapping, and + * the size check is what keeps a short object off the port -- mmap() + * succeeds on one, and the router faults on the first access past its last + * page. */ #include @@ -68,7 +74,7 @@ nxt_router_new_port_test(nxt_thread_t *thr) nxt_buf_t buf; nxt_task_t *task; nxt_int_t ret; - nxt_port_t *port; + nxt_port_t *port, *reply_port; nxt_runtime_t *rt, *saved_rt; nxt_event_engine_t engine; nxt_port_recv_msg_t msg; @@ -81,6 +87,7 @@ nxt_router_new_port_test(nxt_thread_t *thr) sock = -1; queue = -1; port = NULL; + reply_port = NULL; task = thr->task; task->thread = thr; @@ -128,6 +135,33 @@ nxt_router_new_port_test(nxt_thread_t *thr) */ rt->main_engine = &engine; + /* + * The port the message arrived on. An application NEW_PORT that answers + * a pending start carries a stream, and the handler dispatches such a + * message through nxt_port_rpc_handler(), which reads this port; the + * fixture registers no stream, so the dispatch is a lookup that finds + * nothing and returns. + * + * The stream is what keeps the path under test short. With stream 0 a + * handler that took the queue would run on into the main app port lookup + * -- an assertion the fixture cannot satisfy -- and then into a PORT_ACK + * written through the queue it just took, faulting on the short mapping. + * Either would kill the binary instead of failing the checks below. + */ + reply_port = nxt_runtime_process_port_create(task, rt, nxt_pid, 0, + NXT_PROCESS_ROUTER); + if (nxt_slow_path(reply_port == NULL)) { + goto done; + } + + /* + * nxt_port_new() zeroes the port, and 0 is a descriptor teardown must + * not close. + */ + reply_port->pair[0] = -1; + reply_port->pair[1] = -1; + reply_port->socket.fd = -1; + /* * A short queue: mmap() of the whole object over it would succeed and * fault later, so the size check has to refuse it here. @@ -166,8 +200,8 @@ nxt_router_new_port_test(nxt_thread_t *thr) msg.fd[0] = sock; msg.fd[1] = queue; - /* stream 0: the handler returns after the refusal, without RPC. */ - msg.port_msg.stream = 0; + msg.port = reply_port; + msg.port_msg.stream = 1; msg.port_msg.pid = new_port.pid; nxt_router_new_port_handler(task, &msg); @@ -196,6 +230,17 @@ nxt_router_new_port_test(nxt_thread_t *thr) sock = -1; port = msg.u.new_port; + /* + * The point of the refusal: the short object must not be on the port. + * The descriptor checks above say only that nothing leaked, and would + * hold just as well for a handler that mapped the object and kept it. + */ + if (port->queue != NULL) { + nxt_log_alert(thr->log, "router new port test: a refused queue was " + "installed on the port"); + goto done; + } + ret = NXT_OK; nxt_log_error(NXT_LOG_NOTICE, thr->log, "router new port test passed"); @@ -223,6 +268,11 @@ nxt_router_new_port_test(nxt_thread_t *thr) nxt_runtime_port_remove(task, port); } + if (reply_port != NULL) { + nxt_port_close(task, reply_port); + nxt_runtime_port_remove(task, reply_port); + } + thr->runtime = saved_rt; thr->engine = NULL; From 0c06b262e0b63e0157271c71fac9f8df96a6140e Mon Sep 17 00:00:00 2001 From: Paavo-Einari Kaipila Date: Wed, 29 Jul 2026 14:14:26 +0300 Subject: [PATCH 036/391] feat(java): update tomcat jars to 9.0.120 --- auto/modules/java | 2 +- auto/modules/java_jar.sha512 | 27 +++++++++------------------ test/unit/applications/lang/java.py | 2 +- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/auto/modules/java b/auto/modules/java index 30bbd30d7..c03edcffd 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -238,7 +238,7 @@ cat << END > $NXT_JAVA_JARS static const char *nxt_java_system_jars[] = { END -NXT_TOMCAT_VERSION=9.0.110 +NXT_TOMCAT_VERSION=9.0.120 NXT_JAR_VERSION=$NXT_TOMCAT_VERSION diff --git a/auto/modules/java_jar.sha512 b/auto/modules/java_jar.sha512 index 29e0f6199..a786ce956 100644 --- a/auto/modules/java_jar.sha512 +++ b/auto/modules/java_jar.sha512 @@ -3,21 +3,12 @@ ab441acf5551a7dc81c353eaccb3b3df9e89a48987294d19e39acdb83a5b640fcdff7414cee29f5b 609516a5c3b7fa9e256b768124d3c822f28ce426687a479eb57b0f2f6dc82ee6c3560f8e790394b71feebc409eda78a04d6c03254bb41127506ffcf6b7905a61 jetty-http-9.4.58.v20250814.jar 5f2b1efe5f6279ed9c3a25d8c3a6413b195a6957500752cb7b59fc8e8bc43d01eeb2e97b619bfb74ff5ec96cd8893ba1d9d372f8a226ff197d3b452febafa146 jetty-server-9.4.58.v20250814.jar c4505826e68be8d8736eba5cd29fd86ada455ee5765f8d46d33085b788eb3295023b167354178441e80c97edde9f988f89bf3959c59ef8aada56e5d863673021 jetty-util-9.4.58.v20250814.jar -65cfee9beb10eb9831af55f039d8084dcd500a9718f4c753acf99fb52d4ab8b1bd70c8a6807a4cae7096466ea344c79d366c4b115d8b1e641bc6c86e49e2c8c3 tomcat-api-9.0.108.jar -b1970b9af36882c351fc0913ca39d33e184dd97274a36d6856a9a52f930ea0e9f3aa04758905ee7b416526322fe06e846c7f041fcbbeb561be82cc0a30bed647 tomcat-api-9.0.110.jar -772f68efbd68d010eb72d043c115364d65083c9a95887df79bf26feed6aa332348424ec3fbab9155dab508c11a40a31f1d1aea1205d04a180b0619497628ed32 tomcat-el-api-9.0.108.jar -1957abb8d4bf8e51d98a9a4064b157d00d26d0b8fd9513669745c3e61645d9dcc07a570305bd22ca1f0d1685efb8e652b17aa2e6a30cac8f136fda9e15f1baf7 tomcat-el-api-9.0.110.jar -fee3afbb966efdbe49347f1942a3ba5aa275e7193de6a14a0355f49a8138b255f4123189435613976e18c55cb37f4f53e5bc8dd1b9cc178551b4fb0978e545a4 tomcat-jasper-9.0.108.jar -e482fd98ce80176b106a726e2d54506a40a9add016549b35f1f5faff4a95a3a14e5f2ca471ad458a0907064c983693c48a1146650bd1144b1f2b624f37b11d02 tomcat-jasper-9.0.110.jar -93c6ac9876599a1a7c7dfc09458edcc109de0748db77514b765132814d80621a4bca431378acc916306e928119cea1157288ff81f8d88fd1661366aded8213b2 tomcat-jasper-el-9.0.108.jar -d461d63ecaa824298bc72983297bd1f0b108d20c182675dcdd44090a5390cc8889c7a50e50d03d7367939cfc77a33128372ddd6907387ee6d84789e457aa9af7 tomcat-jasper-el-9.0.110.jar -c3bef13c90043fa179c3b7ab599c7ab227e5bf28132f314049222096a1af80b34fc28737f40973c0dc4f10f5899da524fefafc5a97981b0f18e87062a5041766 tomcat-jsp-api-9.0.108.jar -8ab91979e09581009d1fa26b2a99dae59454791095015d818a6db44fe383935a1758d705c405fe7de3633561546d6a6417005d7f9a4e6cb96a818520bf67659b tomcat-jsp-api-9.0.110.jar -3ea930f5da6f2e4ba90208ef4cede22decf737aad252f9c74dcf4ffaa7351ab52e153b31f41fe64fc0c7a1bae626e1e1da2de679975583740975cce4391c67d9 tomcat-juli-9.0.108.jar -313900c4b8c65e232da9431f7b1e2745c81d2a6ece7be06d5bc51da3b5c321afc0080959b48e47a84f99558ecfbbbc642685cb171d2bac85c86d0cb50cfdc103 tomcat-juli-9.0.110.jar -a1e93a66ffe517baa37878d085b18537c18284c8ddf3872d9b4c2ef0d6579c68f0eac57d5d15303cde485b3286bafd20e1b03136e9c86712002a31edd6081052 tomcat-servlet-api-9.0.108.jar -6cce38b6e786538ce09d0098bc75923cea9a08b726dfcdac66625c8d4b103c4d546a037351008d95be026c4ceb99e7db1ee9d605bb5e13a78eff0431e8f7e03c tomcat-servlet-api-9.0.110.jar -cfcabcd79c56f33415b8f4095ac894c0dfa5f392fd272e2e66d61d67051b1eac026740dd21657179a651fa0eeb2f39351781771b0e38799ab1fdf277ebda9883 tomcat-util-9.0.108.jar -f1d183cf8568cab72dcb72dbe052656b448f95c546288237c8548577e51e975dfb371e8152557d4b094ed98035a1bfc5de6edbeca09e5751ca56c9677ae792e5 tomcat-util-9.0.110.jar -0cd88e452ead14c89cd9c70b965795f5793355bc61a0cbf58355585afedcc8a24cb62968da757e253ef1b119021a543aea0058cf9fd5ab152bc185e338c3d15f tomcat-util-scan-9.0.108.jar -0059c8f5bb338d4e4492f15537c9ec9cb337472c878d46b32dbc5811fe517f35917358a23777b2e2eea7f6c9b5c21d07532ddc25f9a82defd73c3c1e11800ada tomcat-util-scan-9.0.110.jar +b8e6b0770b8f2bf53e9aba2f2f0081596e3c6f6709cce42ac4a1559491cc7c68801683057a5397c2725b49a72a521a957acf03eefc9c561d60f7c09923135801 tomcat-api-9.0.120.jar +09205e4b7689154a61d1b2464659b47e9ca3d04775876264416845c52518c9b7565eb01c950f514b8287a649b1019b23fcc4040d0ad09d73e071d6a6550b8b7a tomcat-el-api-9.0.120.jar +a8d22d6e6ca6394919af0abed5dd385bb5000a1844b6185712beb080bee0bc07bb7161b312c2ef17202437ed7f716d888093af04e3e90819e5e12f4147a5327d tomcat-jasper-9.0.120.jar +0733e3950f6fba1b9f3680115a7b8f7739d01b7892e3e96158f9bb95d0fd8c047a918d31f8006ace19a1df1d93c1666f0382adacfa2c5a2ee0648ea63fdd1acd tomcat-jasper-el-9.0.120.jar +81265bd5a58817c95a192b30b740e7f930cb999e5b32492ed5d242bd64aad87e74f0c6dd4f0fa3c7e12b84ad0b7133f5bcb60de049aa6e3616e13bf48881911d tomcat-jsp-api-9.0.120.jar +b95b6f64a325cb302ef6dc7d98d1633bd0ec502c89bd917e9c42135711e02f899eb98fe4607acc5791627d52b548f6917485e2fbac9eb37e39e449d90cec1b5b tomcat-juli-9.0.120.jar +288041872802c7f975685f7fcc78478538d8fd8cc997f973c50e5ffa199d2de963cd2f256f05283d1138be8f46521945bb54dc3f4346cd10cd9145b60d33c08a tomcat-servlet-api-9.0.120.jar +389a187ae1b3e0601d1f28bfe1890b225e6e63aa53a16a1328cf7c79eaee685ba85462bc6fc8997d79b8c1dc8783bf2bec720f1ac50f802c1e36c65239fd534f tomcat-util-9.0.120.jar +8069bebcb4ca3828d055ee2916b03fe27953556000c1e790dee8bab44aec81f5bcea1f0e4ea3cdf99522e4b4b9ec84d56e96f1869114b44ae3b90c006bdd4c0b tomcat-util-scan-9.0.120.jar diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index bd510fd47..135aca4a5 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -53,7 +53,7 @@ def prepare_env(self, script): os.makedirs(classes_path) classpath = ( - f'{option.current_dir}/build/tomcat-servlet-api-9.0.110.jar' + f'{option.current_dir}/build/tomcat-servlet-api-9.0.120.jar' ) ws_jars = glob.glob( From 7156c3d636388ccfcebde28673d542c7ff503ae4 Mon Sep 17 00:00:00 2001 From: Paavo-Einari Kaipila Date: Wed, 29 Jul 2026 14:15:45 +0300 Subject: [PATCH 037/391] feat(java): update classgraph jar to 4.8.186 --- auto/modules/java | 2 +- auto/modules/java_jar.sha512 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto/modules/java b/auto/modules/java index c03edcffd..5f006a902 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -297,7 +297,7 @@ NXT_JAR_NAME=jetty-http . auto/modules/java_get_jar NXT_JAR_NAME=classgraph -NXT_JAR_VERSION=4.8.181 +NXT_JAR_VERSION=4.8.186 NXT_JAR_NAMESPACE=io/github/classgraph/ . auto/modules/java_get_jar diff --git a/auto/modules/java_jar.sha512 b/auto/modules/java_jar.sha512 index a786ce956..343a7d003 100644 --- a/auto/modules/java_jar.sha512 +++ b/auto/modules/java_jar.sha512 @@ -1,4 +1,4 @@ -56ddba7508c2e56f710af1d4c619ccdfda3acc378cfde71d7e5a13b62c1388b424fa91133e528edc16e9619093fb888b2ec7aaa9721d8e14e101ee2a4a00877a classgraph-4.8.181.jar +1e750bb08bff586c1023ddf6af9f3987bf060a5adb3cf1b79d1d1c9ca662ebb3e5aaf8a70a158ecb8b00a836884b4d93ea1bcfe116c92a5e613c2c294451fd49 classgraph-4.8.186.jar ab441acf5551a7dc81c353eaccb3b3df9e89a48987294d19e39acdb83a5b640fcdff7414cee29f5b96eaa8826647f1d5323e185018fe33a64c402d69c73c9158 ecj-3.26.0.jar 609516a5c3b7fa9e256b768124d3c822f28ce426687a479eb57b0f2f6dc82ee6c3560f8e790394b71feebc409eda78a04d6c03254bb41127506ffcf6b7905a61 jetty-http-9.4.58.v20250814.jar 5f2b1efe5f6279ed9c3a25d8c3a6413b195a6957500752cb7b59fc8e8bc43d01eeb2e97b619bfb74ff5ec96cd8893ba1d9d372f8a226ff197d3b452febafa146 jetty-server-9.4.58.v20250814.jar From f674c78e7e182f5b8ec6a45ad76438abf07860ce Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 16:42:10 +0200 Subject: [PATCH 038/391] otel: assert the attribute struct's layout instead of describing it The comment above nxt_otel_attr_t claimed the layout had been "verified with static assertions". It had been -- in a scratch file, not in the tree. Ship the assertions instead, so the claim is enforced rather than remembered: four on the field offsets, and one on sizeof. The size assertion is the one that earns its place. A field appended after sval leaves every offset check passing while the struct grows, and these are passed as an array, so a C-side-only trailing field changes the stride C writes with and not the one Rust reads with -- every attribute after the first in a batch would be misread. Both cases were checked by deliberately breaking the struct and confirming the build fails with the right message. The prose the assertions replace was also wrong on the merits. It said the field order was ABI-significant and that reordering could put C and Rust out of step on a 32-bit target. Because the Rust struct is #[repr(C)], both sides lay out under the target's C ABI, so a change mirrored on both sides -- reordering included -- stays in agreement everywhere; a mirrored uint32_t added before ival lands at offset 12 on i386 and 16 on armv7, with C and Rust agreeing on each. What actually breaks the FFI is editing one side only, or pairing types that are not the same size and alignment on every target we ship: C long is 4 bytes on ILP32 where Rust i64 is always 8, and C enum sizes are implementation-defined, which is why key_id and type are uint32_t here rather than nxt_otel_attr_id_t. Also drop a reference to nxt_otel_rs_add_attr from the test comments; the batched entry point replaced it and the single-attribute function no longer exists. The rest of that comment still holds: attribute keys do still travel as literal strings on the wire, since keying them by id across the FFI changed where they live in the process, not the wire format. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013uPosiJqof76yNnasGdgFn --- src/nxt_otel.h | 33 +++++++++++++++++++++++++++++++++ test/test_otel.py | 8 +++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/nxt_otel.h b/src/nxt_otel.h index 52a9ee14f..eec16332f 100644 --- a/src/nxt_otel.h +++ b/src/nxt_otel.h @@ -49,6 +49,26 @@ typedef enum { /* * One span attribute in a batch. Mirrored by nxt_otel_attr_t in * src/otel/src/lib.rs; the layouts must stay identical. + * + * The Rust struct is #[repr(C)], so both sides lay their fields out under + * the target's C ABI: a change mirrored on both sides, reordering included, + * stays in agreement on every target. What breaks the FFI is editing one + * side only, or pairing fields whose types are not the same size and + * alignment everywhere we ship -- C long is 4 bytes on ILP32 where Rust i64 + * is always 8, and C enum sizes are implementation-defined, which is why + * key_id and type are uint32_t here rather than nxt_otel_attr_id_t. + * + * The offsets below are the same on x86-64, armv7 and i386 (the total size + * is not: 32 on 64-bit, 24 on 32-bit, where int64_t needs only 4-byte + * alignment). They are asserted rather than described, so that a field + * added, reordered or appended here fails the build and prompts the matching + * edit on the Rust side, which these assertions cannot see. + * + * The size assertion is what covers a field appended after sval: the offset + * checks alone would all still hold, while the struct grew. That matters + * more than it looks, because these are passed as an array -- a C-side-only + * trailing field changes the stride C writes with and not the one Rust reads + * with, so every attribute after the first would be misread. */ typedef struct { uint32_t key_id; @@ -57,6 +77,19 @@ typedef struct { nxt_str_t sval; } nxt_otel_attr_t; +nxt_static_assert(offsetof(nxt_otel_attr_t, key_id) == 0, + "nxt_otel_attr_t layout drifted from src/otel/src/lib.rs"); +nxt_static_assert(offsetof(nxt_otel_attr_t, type) == 4, + "nxt_otel_attr_t layout drifted from src/otel/src/lib.rs"); +nxt_static_assert(offsetof(nxt_otel_attr_t, ival) == 8, + "nxt_otel_attr_t layout drifted from src/otel/src/lib.rs"); +nxt_static_assert(offsetof(nxt_otel_attr_t, sval) == 2 * sizeof(uint32_t) + + sizeof(int64_t), + "nxt_otel_attr_t layout drifted from src/otel/src/lib.rs"); +nxt_static_assert(sizeof(nxt_otel_attr_t) == offsetof(nxt_otel_attr_t, sval) + + sizeof(nxt_str_t), + "nxt_otel_attr_t grew a field Rust does not know about"); + #if (NXT_HAVE_OTEL) extern void nxt_otel_rs_send_trace(void *trace); diff --git a/test/test_otel.py b/test/test_otel.py index a526b226e..2d1120b32 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -202,9 +202,11 @@ def test_otel_span_exported_with_service_name(tmp_path, protocol): with open(dump, 'rb') as f: body = f.read() assert b'FreeUnit' in body, 'exported span must carry service.name=FreeUnit' - # Semconv span attributes (1.35.6): recorded via nxt_otel_rs_add_attr, - # not as the old free-form span events. The attribute *keys* travel as - # literal strings in the OTLP protobuf payload. + # Semconv span attributes: recorded via nxt_otel_rs_add_attrs, not as + # the old free-form span events. The attribute *keys* still travel as + # literal strings in the OTLP protobuf payload -- keying them by id + # across the FFI changed where they live in the process, not the wire + # format. assert b'http.request.method' in body, 'span must carry semconv method attr' assert b'url.path' in body, 'span must carry semconv url.path attr' assert b'http.response.status_code' in body, 'span must carry status attr' From 0e70f0480e961358fd34f9683b1959b505000dfd Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 27 Aug 2026 16:05:17 +0200 Subject: [PATCH 039/391] tls: drop the pre-1.1.0 compatibility code and declare the 1.1.1 floor auto/ssltls asked only whether an OpenSSL library was present, and both TLS files carried branches for versions below 1.1.0, so the tree read as though 1.0.x were supported. It has not built since 990ea6ca, which made nxt_openssl_library_init() call OpenSSL_version_num() and OpenSSL_version() unconditionally. Both replaced the pre-1.1.0 SSLeay spellings and no shim was left behind, so against a 1.0.x header the build fails at implicit declaration under the default -Werror, and at link regardless. The floor has been 1.1.0 for four months without anything saying so. Removed, all of it reachable only below 1.1.0: - the locking callbacks and their forward declarations: nxt_openssl_locks, nxt_openssl_locks_init(), nxt_openssl_lock(), nxt_openssl_thread_id(), nxt_openssl_locks_free(), and the duplicate nxt_openssl_library_free() that came with them. The empty stub is now unconditional. - the SSL_load_error_strings()/OPENSSL_config()/SSL_library_init() init arm. OPENSSL_init_ssl() is now unconditional. - the loop stripping the gzip compression method, guarded by #ifndef SSL_OP_NO_COMPRESSION. That macro has existed since 1.0.0, so the guard could not be true even before the floor moved. - the two ASN1_STRING_data() arms in the subject-alt-name loops. Left alone: the RAND_keep_random_devices_open() workaround scoped to exactly 1.1.1, and the 3.0+ ERR_get_error_all() split. Both distinguish versions that are still supported. The floor is now checked where the library is, and it is 1.1.1 rather than the 1.1.0 the code could technically build against. That is a support decision, not a build fact: 1.1.1 is EOL upstream but vendor-maintained on RHEL 8, Amazon Linux 2 and Debian 11, all still in pkg/eol.json's OS matrix into 2027-2032, and pkg/rpm builds against openssl11-devel there. 1.1.0 is maintained by nobody and shipped by nothing in that matrix. auto/ssltls's presence probe compiles against far older releases as well, so it never said anything about usability -- and that is now deliberate, so that a too-old library reaches the floor check and is named there rather than reported as no OpenSSL at all. It is spelled with symbols every OpenSSL has and none deprecates, SSL_CTX_free() and ERR_get_error(), one per library, rather than the SSL_library_init() compat macro that is absent from OPENSSL_NO_DEPRECATED builds. A #error on OPENSSL_VERSION_NUMBER fails configure with a message naming the requirement, instead of leaving it to be discovered as an implicit declaration in the middle of a build. Verified in both directions -- configure reports "checking for OpenSSL 1.1.1 or later ... found" against 3.5.5, and raising the threshold makes it exit 1 with the error, so the check is not one that cannot fail. Builds and passes the C suite against OpenSSL 3.5.5. --- auto/ssltls | 37 ++++++++++++- src/nxt_cert.c | 4 -- src/nxt_openssl.c | 134 ---------------------------------------------- 3 files changed, 36 insertions(+), 139 deletions(-) diff --git a/auto/ssltls b/auto/ssltls index b275e6140..f54aab63e 100644 --- a/auto/ssltls +++ b/auto/ssltls @@ -22,9 +22,11 @@ if [ $NXT_OPENSSL = YES ]; then nxt_feature_incs= nxt_feature_libs="-lssl -lcrypto" nxt_feature_test="#include + #include int main(void) { - SSL_library_init(); + SSL_CTX_free(NULL); + (void) ERR_get_error(); return 0; }" . auto/feature @@ -34,6 +36,39 @@ if [ $NXT_OPENSSL = YES ]; then NXT_TLS=YES NXT_OPENSSL_LIBS="$nxt_feature_libs" + # The probe above uses only symbols that every OpenSSL provides and + # none of them deprecates, so it compiles against a too-old library + # as well; that is deliberate, so that the floor probe below is the + # one to report a too-old library by name instead of configure + # bailing out with a misleading "no OpenSSL library found". The + # floor is not a build fact but a support decision: 1.1.1 is + # vendor-maintained on RHEL 8, Amazon Linux 2 and Debian 11, all + # still in the OS matrix (pkg/eol.json), and pkg/rpm builds against + # openssl11-devel there. + # Checking it here fails configure with a clear message instead of + # leaving a too-old library to surface as an implicit declaration + # deep in the build. + nxt_feature="OpenSSL 1.1.1 or later" + nxt_feature_name= + nxt_feature_run=no + nxt_feature_test="#include + + #if OPENSSL_VERSION_NUMBER < 0x1010100fL + #error OpenSSL 1.1.1 or later is required. + #endif + + int main(void) { + return 0; + }" + . auto/feature + + if [ $nxt_found = no ]; then + $echo + $echo $0: error: OpenSSL 1.1.1 or later is required. + $echo + exit 1; + fi + nxt_feature="OpenSSL version" nxt_feature_name=NXT_HAVE_OPENSSL_VERSION nxt_feature_run=value diff --git a/src/nxt_cert.c b/src/nxt_cert.c index ece5e9fd5..c03d63749 100644 --- a/src/nxt_cert.c +++ b/src/nxt_cert.c @@ -812,11 +812,7 @@ nxt_cert_alt_names_details(nxt_mp_t *mp, STACK_OF(GENERAL_NAME) *alt_names) } str.length = ASN1_STRING_length(name->d.dNSName); -#if OPENSSL_VERSION_NUMBER > 0x10100000L str.start = (u_char *) ASN1_STRING_get0_data(name->d.dNSName); -#else - str.start = ASN1_STRING_data(name->d.dNSName); -#endif ret = nxt_conf_set_element_string_dup(array, mp, i++, &str); if (nxt_slow_path(ret != NXT_OK)) { diff --git a/src/nxt_openssl.c b/src/nxt_openssl.c index 7902640da..dbceb16b7 100644 --- a/src/nxt_openssl.c +++ b/src/nxt_openssl.c @@ -55,12 +55,6 @@ typedef enum { static nxt_int_t nxt_openssl_library_init(nxt_task_t *task); static void nxt_openssl_library_free(nxt_task_t *task); -#if OPENSSL_VERSION_NUMBER < 0x10100004L -static nxt_int_t nxt_openssl_locks_init(void); -static void nxt_openssl_lock(int mode, int type, const char *file, int line); -static unsigned long nxt_openssl_thread_id(void); -static void nxt_openssl_locks_free(void); -#endif static nxt_int_t nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp, nxt_tls_init_t *tls_init, nxt_bool_t last); static nxt_int_t nxt_openssl_chain_file(nxt_task_t *task, SSL_CTX *ctx, @@ -138,55 +132,13 @@ nxt_openssl_library_init(nxt_task_t *task) return NXT_OK; } -#if OPENSSL_VERSION_NUMBER >= 0x10100003L - OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL); -#else - { - nxt_int_t ret; - - SSL_load_error_strings(); - - OPENSSL_config(NULL); - - /* - * SSL_library_init(3): - * - * SSL_library_init() always returns "1", - * so it is safe to discard the return value. - */ - (void) SSL_library_init(); - - ret = nxt_openssl_locks_init(); - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } - } - -#endif - nxt_openssl_version = OpenSSL_version_num(); nxt_log(task, NXT_LOG_INFO, "%s, %xl", OpenSSL_version(OPENSSL_VERSION), nxt_openssl_version); -#ifndef SSL_OP_NO_COMPRESSION - { - /* - * Disable gzip compression in OpenSSL prior to 1.0.0 - * version, this saves about 522K per connection. - */ - int n; - STACK_OF(SSL_COMP) *ssl_comp_methods; - - ssl_comp_methods = SSL_COMP_get_compression_methods(); - - for (n = sk_SSL_COMP_num(ssl_comp_methods); n != 0; n--) { - (void) sk_SSL_COMP_pop(ssl_comp_methods); - } - } -#endif index = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL); @@ -202,94 +154,12 @@ nxt_openssl_library_init(nxt_task_t *task) } -#if OPENSSL_VERSION_NUMBER >= 0x10100003L - -static void -nxt_openssl_library_free(nxt_task_t *task) -{ -} - -#else - -static nxt_thread_mutex_t *nxt_openssl_locks; - -static nxt_int_t -nxt_openssl_locks_init(void) -{ - int i, n; - nxt_int_t ret; - - n = CRYPTO_num_locks(); - - nxt_openssl_locks = OPENSSL_malloc(n * sizeof(nxt_thread_mutex_t)); - if (nxt_slow_path(nxt_openssl_locks == NULL)) { - return NXT_ERROR; - } - - for (i = 0; i < n; i++) { - ret = nxt_thread_mutex_create(&nxt_openssl_locks[i]); - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } - } - - CRYPTO_set_locking_callback(nxt_openssl_lock); - - CRYPTO_set_id_callback(nxt_openssl_thread_id); - - return NXT_OK; -} - - -static void -nxt_openssl_lock(int mode, int type, const char *file, int line) -{ - nxt_thread_mutex_t *lock; - - lock = &nxt_openssl_locks[type]; - - if ((mode & CRYPTO_LOCK) != 0) { - (void) nxt_thread_mutex_lock(lock); - - } else { - (void) nxt_thread_mutex_unlock(lock); - } -} - - -static u_long -nxt_openssl_thread_id(void) -{ - return (u_long) nxt_thread_handle(); -} - - static void nxt_openssl_library_free(nxt_task_t *task) { - nxt_openssl_locks_free(); } -static void -nxt_openssl_locks_free(void) -{ - int i, n; - - n = CRYPTO_num_locks(); - - CRYPTO_set_locking_callback(NULL); - - for (i = 0; i < n; i++) { - nxt_thread_mutex_destroy(&nxt_openssl_locks[i]); - } - - OPENSSL_free(nxt_openssl_locks); -} - -#endif - - static nxt_int_t nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp, nxt_tls_init_t *tls_init, nxt_bool_t last) @@ -875,11 +745,7 @@ nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, } str.length = ASN1_STRING_length(name->d.dNSName); -#if OPENSSL_VERSION_NUMBER > 0x10100000L str.start = (u_char *) ASN1_STRING_get0_data(name->d.dNSName); -#else - str.start = ASN1_STRING_data(name->d.dNSName); -#endif domain.start = nxt_mp_nget(mp, str.length); if (nxt_slow_path(domain.start == NULL)) { From 1fe09f224a36b00b85b78b32d5e2a95e4df547a9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 24 Aug 2026 01:53:56 +0200 Subject: [PATCH 040/391] comments: drop a stale example call and fix a padding-size claim nxt_router_response_error_handler() carried a commented-out nxt_router_msg_cancel() call with three arguments; the function has taken two (task, req_rpc_data) since it started deriving msg_info internally, so the example no longer matches the code it illustrates. Drop the stale line and keep the still-open TODO above it. nxt_test_build.h's stub struct signalfd_siginfo claims its pad[8] brings the struct to 128 bytes; the preceding fields already sum to 80, so pad[8] only reaches 88. The size is otherwise harmless here -- this struct exists only to type-check the epoll build path on non-Linux hosts and is never used for a real signalfd(2) read -- but the comment's arithmetic was wrong, so restate what the padding is for instead of a number that doesn't add up. --- src/nxt_router.c | 1 - src/nxt_test_build.h | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 56e265be4..58488c3f3 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -4906,7 +4906,6 @@ nxt_router_response_error_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, req_rpc_data->rpc_cancel = 0; /* TODO cancel message and return if cancelled. */ - // nxt_router_msg_cancel(task, &req_rpc_data->msg_info, req_rpc_data->stream); if (req_rpc_data->request != NULL) { nxt_http_request_error(task, req_rpc_data->request, diff --git a/src/nxt_test_build.h b/src/nxt_test_build.h index 6513e748f..2c563f2c3 100644 --- a/src/nxt_test_build.h +++ b/src/nxt_test_build.h @@ -74,8 +74,12 @@ struct signalfd_siginfo { uint64_t ssi_stime; /* System CPU time consumed (SIGCHLD) */ uint64_t ssi_addr; /* Address that generated signal (for hardware-generated signals) */ - uint8_t pad[8]; /* Pad size to 128 bytes (allow for - additional fields in the future) */ + uint8_t pad[8]; /* Padding; the real kernel struct pads + to 128 bytes, but the exact size does + not matter here since this is a + syntax-check-only stub for non-Linux + test builds, never used for a real + signalfd(2) read */ }; From 254add2786a257c40ffc7ac00a88690cdaf0b3f6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 18:34:43 +0200 Subject: [PATCH 041/391] port: stop describing a queueless port as a socket fallback Three separate places have stated that a port whose queue could not be mapped "falls back" to plain socket writes and keeps working: the log message e8b7f644 added to nxt_port_process_ready_handler() ("falling back to the socket"), the comment that survived its rewording in 6757c ("on the socket if it had no queue yet"), and a justification offered during the PR #212 review. The claim is false for every libunit application, and the same PR later proved it empirically: a worker behind a queueless port wedged on the first CHANGE_FILE with "too many port socket messages". The property, in both directions: nxt_port_socket_write2() emits the READ_SOCKET marker only under port->queue != NULL, and nxt_unit_ctx_port_recv() hands a socket message to the application only after such a marker was dequeued from the port's queue. A queueless sender cannot emit the marker, so its messages are suspended undelivered on the worker side and the second one fails the worker's context. Since nxt_port_send_port() forwards queue_fd == -1, every peer ends up with the same unreachable copy: QUIT can never arrive, while requests keep flowing through the shared app queue -- a worker that serves but cannot be stopped. Only PROCESS_READY from libunit carries a queue descriptor at all (native processes pass -1), so the mapping arm runs exactly for the peers the fallback does not exist for. Say so where the code claimed otherwise. The ready handler now alerts that the process cannot be reached when the failed mapping leaves the port with no queue, and keeps the mild warning for the repeated-READY case that retains a working queue; nxt_main_new_port_handler() logs the consequence instead of continuing in silence. No behaviour changes: failing the start would be the honest reaction to a genuine mmap failure, but PROCESS_READY is unauthenticated and refusing it hands a forged message a way to kill a legitimate start, so that half waits for sender authentication. Issue #231 tracks the property and that follow-up. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Asbr2xiUkx9axPtYy9Km2c --- src/nxt_main_process.c | 13 +++++++++++++ src/nxt_port.c | 40 +++++++++++++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index ad2cec617..fdb52aa05 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -429,6 +429,19 @@ nxt_main_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_fast_path(mem != NULL)) { port->queue = mem; + + } else { + /* + * Not a fallback to the socket: a libunit process delivers a + * socket message only after dequeuing the READ_SOCKET marker + * that only a queue-holding sender emits, so everything main + * sends on this port -- CHANGE_FILE on log rotation, QUIT on + * shutdown -- would be suspended undelivered, and the second + * message fails the worker's context. See issue #231. + */ + nxt_alert(task, "cannot map the queue of port %PI:%d; the " + "process cannot be reached on this port from main", + port->pid, (int) port->id); } } diff --git a/src/nxt_port.c b/src/nxt_port.c index c4653f99a..e1abd394a 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -550,14 +550,40 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } else { /* - * A refused queue leaves the port as it was, as a failed - * mapping always did -- on the socket if it had no queue yet, - * on the one it already has otherwise. Refusing the message - * outright would let a forged PROCESS_READY carrying a bad - * descriptor keep the process from ever being marked ready. + * A refused queue leaves the port as it was. For a repeated + * PROCESS_READY that is the queue it already has, and the port + * keeps working. For the first one it is no queue at all, and + * that is not a fallback to the socket: only libunit attaches + * a queue descriptor to PROCESS_READY, and a libunit process + * delivers a socket message only after dequeuing the + * READ_SOCKET marker that nxt_port_socket_write2() emits under + * port->queue != NULL. A queueless sender cannot emit the + * marker, so its messages are suspended undelivered on the + * worker side, and the second one fails the worker's context + * ("too many port socket messages"). The broadcast below + * forwards queue_fd == -1, so every peer holds the same + * unreachable copy; a QUIT can never arrive, while requests + * still flow through the shared app queue. See issue #231. + * + * The message is still not refused outright: it is not + * authenticated, and refusing it would let a forged + * PROCESS_READY carrying a bad descriptor keep the process + * from ever being marked ready. Failing the start is the + * honest reaction to a genuine mmap failure, but it hands the + * same forgery a way to kill a legitimate start, so it has to + * wait for sender authentication (see the lookup comment + * above). */ - nxt_log(task, NXT_LOG_WARN, "process %PI ready: cannot map the " - "queue, leaving the port as it was", msg->port_msg.pid); + if (port->queue == NULL) { + nxt_alert(task, "process %PI ready: cannot map the queue; " + "the process cannot be reached on this port", + msg->port_msg.pid); + + } else { + nxt_log(task, NXT_LOG_WARN, "process %PI ready: cannot map " + "the replacement queue, keeping the current one", + msg->port_msg.pid); + } } } From 2ad5d46be019fa774674142680d8cfbea85797e6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 19:31:18 +0200 Subject: [PATCH 042/391] java: update tomcat jars to 9.0.121 for CVE-2026-66299 Apache Tomcat 9.0.121 fixes CVE-2026-66299, per https://tomcat.apache.org/security-9.html#Fixed_in_Apache_Tomcat_9.0.121: "Low: DoS in WebSocket chat example CVE-2026-66299 -- The WebSocket chat example provided an unbounded buffer for undelivered messages. A maliciously slow client could cause the buffer to grow continuously, eventually leading to an memory exhaustion and failure of the Tomcat process. Affects: 9.0.89 to 9.0.120" 9.0.121 is the newest 9.0.x published on Maven Central (https://repo1.maven.org/maven2/org/apache/tomcat/tomcat-catalina/maven-metadata.xml). All nine jars the build downloads were fetched from https://repo1.maven.org/maven2/org/apache/tomcat//9.0.121/ and each recorded SHA512 was verified against Maven Central's .sha512 sidecar for the same file. Follow-up to #189. Co-Authored-By: Claude Opus --- auto/modules/java | 2 +- auto/modules/java_jar.sha512 | 18 +++++++++--------- test/unit/applications/lang/java.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/auto/modules/java b/auto/modules/java index 5f006a902..e6d7e3448 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -238,7 +238,7 @@ cat << END > $NXT_JAVA_JARS static const char *nxt_java_system_jars[] = { END -NXT_TOMCAT_VERSION=9.0.120 +NXT_TOMCAT_VERSION=9.0.121 NXT_JAR_VERSION=$NXT_TOMCAT_VERSION diff --git a/auto/modules/java_jar.sha512 b/auto/modules/java_jar.sha512 index 343a7d003..421ea5350 100644 --- a/auto/modules/java_jar.sha512 +++ b/auto/modules/java_jar.sha512 @@ -3,12 +3,12 @@ ab441acf5551a7dc81c353eaccb3b3df9e89a48987294d19e39acdb83a5b640fcdff7414cee29f5b 609516a5c3b7fa9e256b768124d3c822f28ce426687a479eb57b0f2f6dc82ee6c3560f8e790394b71feebc409eda78a04d6c03254bb41127506ffcf6b7905a61 jetty-http-9.4.58.v20250814.jar 5f2b1efe5f6279ed9c3a25d8c3a6413b195a6957500752cb7b59fc8e8bc43d01eeb2e97b619bfb74ff5ec96cd8893ba1d9d372f8a226ff197d3b452febafa146 jetty-server-9.4.58.v20250814.jar c4505826e68be8d8736eba5cd29fd86ada455ee5765f8d46d33085b788eb3295023b167354178441e80c97edde9f988f89bf3959c59ef8aada56e5d863673021 jetty-util-9.4.58.v20250814.jar -b8e6b0770b8f2bf53e9aba2f2f0081596e3c6f6709cce42ac4a1559491cc7c68801683057a5397c2725b49a72a521a957acf03eefc9c561d60f7c09923135801 tomcat-api-9.0.120.jar -09205e4b7689154a61d1b2464659b47e9ca3d04775876264416845c52518c9b7565eb01c950f514b8287a649b1019b23fcc4040d0ad09d73e071d6a6550b8b7a tomcat-el-api-9.0.120.jar -a8d22d6e6ca6394919af0abed5dd385bb5000a1844b6185712beb080bee0bc07bb7161b312c2ef17202437ed7f716d888093af04e3e90819e5e12f4147a5327d tomcat-jasper-9.0.120.jar -0733e3950f6fba1b9f3680115a7b8f7739d01b7892e3e96158f9bb95d0fd8c047a918d31f8006ace19a1df1d93c1666f0382adacfa2c5a2ee0648ea63fdd1acd tomcat-jasper-el-9.0.120.jar -81265bd5a58817c95a192b30b740e7f930cb999e5b32492ed5d242bd64aad87e74f0c6dd4f0fa3c7e12b84ad0b7133f5bcb60de049aa6e3616e13bf48881911d tomcat-jsp-api-9.0.120.jar -b95b6f64a325cb302ef6dc7d98d1633bd0ec502c89bd917e9c42135711e02f899eb98fe4607acc5791627d52b548f6917485e2fbac9eb37e39e449d90cec1b5b tomcat-juli-9.0.120.jar -288041872802c7f975685f7fcc78478538d8fd8cc997f973c50e5ffa199d2de963cd2f256f05283d1138be8f46521945bb54dc3f4346cd10cd9145b60d33c08a tomcat-servlet-api-9.0.120.jar -389a187ae1b3e0601d1f28bfe1890b225e6e63aa53a16a1328cf7c79eaee685ba85462bc6fc8997d79b8c1dc8783bf2bec720f1ac50f802c1e36c65239fd534f tomcat-util-9.0.120.jar -8069bebcb4ca3828d055ee2916b03fe27953556000c1e790dee8bab44aec81f5bcea1f0e4ea3cdf99522e4b4b9ec84d56e96f1869114b44ae3b90c006bdd4c0b tomcat-util-scan-9.0.120.jar +173325a3e237f1f99fb05bbf8569040fb93af0ebadfb73019aaa9640fd351b16cc66a9b69deed8f9f3a945d40413b7ca9633fef7d877778436b341452b0ad653 tomcat-api-9.0.121.jar +aed5789fd28040f5bc88336a3fef1372d6949e2fc912381c7a32594514f18250e7b45f671260932ea04d472675d5f2f35010ef2109dce0416b0706ffc0c701bc tomcat-el-api-9.0.121.jar +3464120da470ac4fd75dad030aa58d5e9a07c36e34287b9febdf5f2025e7642dc43f874095dc0379968a469ffedfef67ae0568ca611d29506fc6175858c5ca76 tomcat-jasper-9.0.121.jar +78d0ec8c83d8733660f62d4ebbb037536cf242fdf3d1111c32a26ecd5100022bd800cd791e0e510cb383119c0bf068b9bf2eaff8cf7f8d5caa0a5c93af61a32c tomcat-jasper-el-9.0.121.jar +88970a8c75f623d157cef417dd0ba0d2ce1000be56c7f205545c087f46c1a9130c54dba49e6926d526a7d74f947d13d020df882fe54c2700385a43e3ca6e7e43 tomcat-jsp-api-9.0.121.jar +19f3e032b914aed600199f5a2e95cf20374daf24fef999586d4710fce54ffb124f8ffadd314c337c327db18bcbfa06c87862b3dc7606a465388aba9208ebca48 tomcat-juli-9.0.121.jar +b74470d50cc6c058cb0a2b28e2b7780f731003fc9d0ad9ea8e53a145ff548ace2b6b952baecc15cf4fd052bec4b80730a9d7c33df721e0fa67998beffeeea790 tomcat-servlet-api-9.0.121.jar +f39189d511dee85c766f5046f53c17e6ff541cbec2d325adf2ebf043997fa29212df1b53effd029a72acab5bb37f130a25a949e5e48983744f862907ce959af1 tomcat-util-9.0.121.jar +a272f0d5dde67ef5468ac7fb8e9b33fb5457b999952931a6329ea767a91001792fd86f2cd358a6152dadba8585f40016519d1cdc135d250f987a1a6d2997b026 tomcat-util-scan-9.0.121.jar diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index 135aca4a5..40fda07e0 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -53,7 +53,7 @@ def prepare_env(self, script): os.makedirs(classes_path) classpath = ( - f'{option.current_dir}/build/tomcat-servlet-api-9.0.120.jar' + f'{option.current_dir}/build/tomcat-servlet-api-9.0.121.jar' ) ws_jars = glob.glob( From d8f41ef9cca9b08aafb415f9ca5ca1eab79ad05b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 24 Aug 2026 01:25:36 +0200 Subject: [PATCH 043/391] tls: move the session ticket key callback onto EVP_MAC OpenSSL 3.0 deprecated HMAC_Init_ex() together with the whole HMAC_CTX flavour of the session ticket key callback, and OpenSSL 5.0 removes the deprecated symbols. Unit hides the warnings by defining OPENSSL_SUPPRESS_DEPRECATED, which keeps the build quiet but does nothing about the removal. The deprecation is wider than the one call: SSL_CTX_set_tlsext_ticket_key_cb() is a macro over SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB, and that control is itself declared under OPENSSL_NO_DEPRECATED_3_0. The feature guard around the whole ticket path tested for that control, so when the deprecated symbols go away Unit would not fail to build -- it would quietly take the "not supported with this version of OpenSSL library" branch and stop serving session tickets. The guard therefore moves to NXT_OPENSSL_TICKET_KEY_CB, picked from the callback flavour that actually exists. On 3.0+ the callback is registered with SSL_CTX_set_tlsext_ticket_key_evp_cb(), introduced in 3.0 precisely as the replacement, and receives an EVP_MAC_CTX that OpenSSL has already created as an HMAC context. Initialising it needs the same two inputs HMAC_Init_ex() took: EVP_MAC_init() is handed ticket[i].hmac_key and ticket[i].size unchanged, and the digest travels as an OSSL_MAC_PARAM_DIGEST parameter naming the same EVP_MD the old code passed. So the key derivation is untouched, the tag length still follows from the digest (32 bytes for SHA-256, 20 with OPENSSL_NO_SHA256), and the tag comparison stays where it always was, inside libssl. The callback's return values -- 0 for an unknown key name, 1, 2 for renew, -1 for failure -- are unchanged, so key rotation behaves identically. Pre-3.0 keeps the HMAC_CTX callback verbatim; only the MAC init and the registration call differ between the two, everything else is shared. --- src/nxt_openssl.c | 77 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/src/nxt_openssl.c b/src/nxt_openssl.c index dbceb16b7..47c6e4ff0 100644 --- a/src/nxt_openssl.c +++ b/src/nxt_openssl.c @@ -31,6 +31,38 @@ typedef struct { } nxt_openssl_conn_t; +#if (NXT_HAVE_OPENSSL_TLSEXT) + +/* + * OpenSSL 3.0 deprecated the HMAC_CTX flavour of the session ticket key + * callback in favour of SSL_CTX_set_tlsext_ticket_key_evp_cb(), which hands + * the callback an EVP_MAC_CTX instead. The deprecation covers the + * SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB control behind the old registration + * macro as well, so the feature guard has to follow the callback flavour: + * once OpenSSL 5.0 drops the deprecated symbols, testing for the control + * would silently turn Session Tickets off. + */ + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + +#include +#include + +#define NXT_OPENSSL_TICKET_KEY_CB 1 +typedef EVP_MAC_CTX nxt_tls_ticket_mac_t; + +#elif defined(SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB) + +#include + +#define NXT_OPENSSL_TICKET_KEY_CB 1 +typedef HMAC_CTX nxt_tls_ticket_mac_t; + +#endif + +#endif /* NXT_HAVE_OPENSSL_TLSEXT */ + + struct nxt_tls_ticket_s { u_char name[16]; u_char hmac_key[32]; @@ -66,8 +98,11 @@ static nxt_int_t nxt_ssl_conf_commands(nxt_task_t *task, SSL_CTX *ctx, #if (NXT_HAVE_OPENSSL_TLSEXT) static nxt_int_t nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init, nxt_mp_t *mp); +#ifdef NXT_OPENSSL_TICKET_KEY_CB static int nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, - unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc); + unsigned char *iv, EVP_CIPHER_CTX *ectx, nxt_tls_ticket_mac_t *hctx, + int enc); +#endif #endif static void nxt_ssl_session_cache(SSL_CTX *ctx, size_t cache_size, time_t timeout); @@ -530,7 +565,7 @@ nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init, goto no_ticket; } -#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB +#ifdef NXT_OPENSSL_TICKET_KEY_CB tickets = nxt_mp_get(mp, sizeof(nxt_tls_tickets_t) + count * sizeof(nxt_tls_ticket_t)); @@ -571,8 +606,13 @@ nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init, } while (i < count); +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + if (SSL_CTX_set_tlsext_ticket_key_evp_cb(ctx, nxt_tls_ticket_key_callback) + == 0) +#else if (SSL_CTX_set_tlsext_ticket_key_cb(ctx, nxt_tls_ticket_key_callback) == 0) +#endif { nxt_openssl_log_error(task, NXT_LOG_ALERT, "Unit was built with Session Tickets support, however, " @@ -601,11 +641,11 @@ nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init, } -#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB +#ifdef NXT_OPENSSL_TICKET_KEY_CB static int nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv, - EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc) + EVP_CIPHER_CTX *ectx, nxt_tls_ticket_mac_t *hctx, int enc) { nxt_uint_t i; nxt_conn_t *c; @@ -613,6 +653,9 @@ nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv, const EVP_CIPHER *cipher; nxt_tls_ticket_t *ticket; nxt_openssl_conn_t *tls; +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + OSSL_PARAM params[2]; +#endif c = SSL_get_ex_data(s, nxt_openssl_connection_index); @@ -685,6 +728,28 @@ nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv, digest = EVP_sha256(); #endif +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + + /* + * The EVP_MAC_CTX handed to this callback is an already created HMAC + * context; it only needs the digest and the key. EVP_MAC_init() takes + * both, so the key material, its length and the digest are exactly the + * ones HMAC_Init_ex() used to be given, and the tag length still follows + * from the digest. + */ + + params[0] = OSSL_PARAM_construct_utf8_string(OSSL_MAC_PARAM_DIGEST, + (char *) EVP_MD_get0_name(digest), 0); + params[1] = OSSL_PARAM_construct_end(); + + if (EVP_MAC_init(hctx, ticket[i].hmac_key, ticket[i].size, params) != 1) { + nxt_openssl_log_error(c->socket.task, NXT_LOG_ALERT, + "EVP_MAC_init() failed"); + return -1; + } + +#else + if (HMAC_Init_ex(hctx, ticket[i].hmac_key, ticket[i].size, digest, NULL) != 1) { @@ -693,10 +758,12 @@ nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv, return -1; } +#endif + return enc; } -#endif /* SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB */ +#endif /* NXT_OPENSSL_TICKET_KEY_CB */ #endif /* NXT_HAVE_OPENSSL_TLSEXT */ From 59ef69f4d0d966d0af3f6361ef8310a7b7f29558 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 24 Aug 2026 01:28:16 +0200 Subject: [PATCH 044/391] tls: read certificate names and validity times without deprecated APIs X509_NAME_get_text_by_NID() is deprecated and cannot express a name entry that contains an embedded NUL: it copies into a caller buffer and stops at the first one, so a certificate crafted with "example.com\0evil.com" reads back as the shorter name. Read the entry directly instead -- X509_NAME_get_index_by_NID(), X509_NAME_get_entry(), then X509_NAME_ENTRY_get_data() -- and take the length from the ASN1_STRING, so the value is handled by length rather than by terminator. The validity times move the same way: X509_get_notBefore() and X509_get_notAfter() are deprecated aliases, and OpenSSL 4.0 drops them. X509_get0_notBefore()/X509_get0_notAfter() have existed since 1.1.0, inside the floor this project now declares, and return a const pointer -- which is all the call site wants, since the value only feeds ASN1_TIME_print(). Both changes then require const on the locals, and that is not uniform across versions: X509_NAME_get_entry() and X509_NAME_ENTRY_get_data() took const arguments from 1.1.0, X509_NAME_get_index_by_NID() only from 3.0, and 4.0 is what constified the return types. Marking the locals const outright builds on 3.x and 4.0 and fails on 1.1.x, so the qualifier is applied per version. Two of the six sites that needed it -- nxt_cert.c and nxt_openssl.c in the certificate name path -- were already there before this series. Co-Authored-By: Claude Opus 5 --- src/nxt_cert.c | 95 +++++++++++++++++++++++++++++++++++++++++------ src/nxt_openssl.c | 83 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 156 insertions(+), 22 deletions(-) diff --git a/src/nxt_cert.c b/src/nxt_cert.c index c03d63749..b8c3df607 100644 --- a/src/nxt_cert.c +++ b/src/nxt_cert.c @@ -19,6 +19,26 @@ #include +/* The buffer X509_NAME_get_text_by_NID() used to be given, less the NUL. */ +#define NXT_CERT_NAME_TEXT_MAX 255 + +/* + * The X509 name accessors were constified in stages, and not together: + * X509_NAME_get_entry() and X509_NAME_ENTRY_get_data() took const arguments + * from 1.1.0, but X509_NAME_get_index_by_NID() only from 3.0, and it is 4.0 + * that constified the values all three return. Qualifying from 3.0 is what + * satisfies every version at once -- an older header takes these pointers + * non-const, and 4.0 hands them back const -- so the boundary is 3.0 rather + * than the 1.1.0 the other guards in this file use. + */ + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#define NXT_X509_CONST const +#else +#define NXT_X509_CONST +#endif + + struct nxt_cert_s { EVP_PKEY *key; nxt_uint_t count; @@ -46,6 +66,8 @@ static int nxt_nxt_cert_pem_suffix(char *pem_str, const char *suffix); static nxt_conf_value_t *nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert); static nxt_conf_value_t *nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer); +static nxt_int_t nxt_cert_name_text(NXT_X509_CONST X509_NAME *x509_name, + int nid, nxt_str_t *str); static nxt_conf_value_t *nxt_cert_alt_names_details(nxt_mp_t *mp, STACK_OF(GENERAL_NAME) *alt_names); static void nxt_cert_buf_completion(nxt_task_t *task, void *obj, void *data); @@ -536,7 +558,7 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) X509 *x509; u_char *end; EVP_PKEY *key; - ASN1_TIME *asn1_time; + const ASN1_TIME *asn1_time; nxt_str_t str; nxt_int_t ret; nxt_uint_t i; @@ -631,7 +653,7 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) return NULL; } - asn1_time = X509_get_notBefore(x509); + asn1_time = X509_get0_notBefore(x509); ret = ASN1_TIME_print(bio, asn1_time); @@ -654,7 +676,7 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) return NULL; } - asn1_time = X509_get_notAfter(x509); + asn1_time = X509_get0_notAfter(x509); ret = ASN1_TIME_print(bio, asn1_time); @@ -692,14 +714,12 @@ typedef struct { static nxt_conf_value_t * nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer) { - int len; - X509_NAME *x509_name; + NXT_X509_CONST X509_NAME *x509_name; nxt_str_t str; - nxt_int_t ret; + nxt_int_t ret, found; nxt_uint_t i, n, count; nxt_conf_value_t *object, *names; STACK_OF(GENERAL_NAME) *alt_names; - u_char buf[256]; static const nxt_cert_nid_t nids[] = { { NID_commonName, nxt_string("common_name") }, @@ -752,19 +772,25 @@ nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer) for (n = 0, i = 0; n != nxt_nitems(nids) && i != count; n++) { - len = X509_NAME_get_text_by_NID(x509_name, nids[n].nid, - (char *) buf, sizeof(buf)); + found = nxt_cert_name_text(x509_name, nids[n].nid, &str); if (n == 1 && names != NULL) { nxt_conf_set_member(object, &alt_names_str, names, i++); } - if (len < 0) { + if (found != NXT_OK) { continue; } - str.length = len; - str.start = buf; + /* + * X509_NAME_get_text_by_NID() used to copy into a 256 byte buffer and + * report the truncated length. Keep that cap: it is what the control + * API has always reported for pathologically long names. + */ + + if (str.length > NXT_CERT_NAME_TEXT_MAX) { + str.length = NXT_CERT_NAME_TEXT_MAX; + } ret = nxt_conf_set_member_string_dup(object, mp, &nids[n].name, &str, i++); @@ -777,6 +803,51 @@ nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer) } +/* + * Return the raw contents of the first entry of x509_name that carries nid, + * or NXT_DECLINED if there is none. + * + * This replaces X509_NAME_get_text_by_NID(), deprecated in OpenSSL 4.0. The + * old function copied the entry verbatim into the caller's buffer; the entry + * accessors hand out the same bytes without the copy, pointing into the + * certificate, which outlives the returned string here. + * + * As before the bytes are neither NUL terminated nor guaranteed free of + * embedded NULs, so the caller must keep treating them as a counted string. + */ + +static nxt_int_t +nxt_cert_name_text(NXT_X509_CONST X509_NAME *x509_name, int nid, + nxt_str_t *str) +{ + int i, len; + NXT_X509_CONST ASN1_STRING *data; + NXT_X509_CONST X509_NAME_ENTRY *entry; + + i = X509_NAME_get_index_by_NID(x509_name, nid, -1); + if (i < 0) { + return NXT_DECLINED; + } + + entry = X509_NAME_get_entry(x509_name, i); + if (nxt_slow_path(entry == NULL)) { + return NXT_DECLINED; + } + + data = X509_NAME_ENTRY_get_data(entry); + + len = ASN1_STRING_length(data); + if (nxt_slow_path(len < 0)) { + return NXT_DECLINED; + } + + str->length = len; + str->start = (u_char *) ASN1_STRING_get0_data(data); + + return NXT_OK; +} + + static nxt_conf_value_t * nxt_cert_alt_names_details(nxt_mp_t *mp, STACK_OF(GENERAL_NAME) *alt_names) { diff --git a/src/nxt_openssl.c b/src/nxt_openssl.c index 47c6e4ff0..5589eccd5 100644 --- a/src/nxt_openssl.c +++ b/src/nxt_openssl.c @@ -17,6 +17,22 @@ #include #include +/* + * The X509 name accessors were constified in stages, and not together: + * X509_NAME_get_entry() and X509_NAME_ENTRY_get_data() took const arguments + * from 1.1.0, but X509_NAME_get_index_by_NID() only from 3.0, and it is 4.0 + * that constified the values all three return. Qualifying from 3.0 is what + * satisfies every version at once -- an older header takes these pointers + * non-const, and 4.0 hands them back const -- so the boundary is 3.0 rather + * than the 1.1.0 the other guards in this file use. + */ + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#define NXT_X509_CONST const +#else +#define NXT_X509_CONST +#endif + typedef struct { SSL *session; @@ -108,6 +124,8 @@ static void nxt_ssl_session_cache(SSL_CTX *ctx, size_t cache_size, time_t timeout); static nxt_uint_t nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, nxt_mp_t *mp); +static nxt_int_t nxt_openssl_name_text(NXT_X509_CONST X509_NAME *x509_name, + int nid, nxt_str_t *str); static nxt_int_t nxt_openssl_bundle_hash_test(nxt_lvlhsh_query_t *lhq, void *data); static nxt_int_t nxt_openssl_bundle_hash_insert(nxt_task_t *task, @@ -788,9 +806,8 @@ static nxt_uint_t nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, nxt_mp_t *mp) { - int len; nxt_str_t domain, str; - X509_NAME *x509_name; + NXT_X509_CONST X509_NAME *x509_name; nxt_uint_t i, n; GENERAL_NAME *name; nxt_tls_bundle_conf_t *bundle; @@ -842,23 +859,22 @@ nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, } else { x509_name = X509_get_subject_name(cert); - len = X509_NAME_get_text_by_NID(x509_name, NID_commonName, - NULL, 0); - if (len <= 0) { + + if (nxt_openssl_name_text(x509_name, NID_commonName, &str) != NXT_OK + || str.length == 0) + { nxt_log(task, NXT_LOG_WARN, "certificate \"%V\" has neither " "Subject Alternative Name nor Common Name", &bundle->name); return NXT_OK; } - domain.start = nxt_mp_nget(mp, len + 1); + domain.start = nxt_mp_nget(mp, str.length); if (nxt_slow_path(domain.start == NULL)) { return NXT_ERROR; } - domain.length = X509_NAME_get_text_by_NID(x509_name, NID_commonName, - (char *) domain.start, - len + 1); - nxt_memcpy_lowcase(domain.start, domain.start, domain.length); + domain.length = str.length; + nxt_memcpy_lowcase(domain.start, str.start, str.length); item = nxt_mp_get(mp, sizeof(nxt_tls_bundle_hash_item_t)); if (nxt_slow_path(item == NULL)) { @@ -886,6 +902,53 @@ nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf, } +/* + * Return the raw contents of the first entry of x509_name that carries nid, + * or NXT_DECLINED if there is none. + * + * This replaces X509_NAME_get_text_by_NID(), deprecated in OpenSSL 4.0. The + * only property of that function the callers ever used was "the bytes of the + * first matching entry, verbatim", which is what the entry accessors give + * directly and without the copy: the returned string points into the + * certificate and stays valid for as long as it does. + * + * The bytes are not NUL terminated and, exactly as before, may contain + * embedded NULs -- the callers must keep handling them with an explicit + * length and never as a C string. + */ + +static nxt_int_t +nxt_openssl_name_text(NXT_X509_CONST X509_NAME *x509_name, int nid, + nxt_str_t *str) +{ + int i, len; + NXT_X509_CONST ASN1_STRING *data; + NXT_X509_CONST X509_NAME_ENTRY *entry; + + i = X509_NAME_get_index_by_NID(x509_name, nid, -1); + if (i < 0) { + return NXT_DECLINED; + } + + entry = X509_NAME_get_entry(x509_name, i); + if (nxt_slow_path(entry == NULL)) { + return NXT_DECLINED; + } + + data = X509_NAME_ENTRY_get_data(entry); + + len = ASN1_STRING_length(data); + if (nxt_slow_path(len < 0)) { + return NXT_DECLINED; + } + + str->length = len; + str->start = (u_char *) ASN1_STRING_get0_data(data); + + return NXT_OK; +} + + static const nxt_lvlhsh_proto_t nxt_openssl_bundle_hash_proto nxt_aligned(64) = { From cbfdf683eca4062019fb9c769cd06c6fb4de808a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 24 Aug 2026 01:45:57 +0200 Subject: [PATCH 045/391] ci: build against OpenSSL 4.0 and run the C tests Unit still calls a fair amount of API that OpenSSL has deprecated, and hides the warnings behind OPENSSL_SUPPRESS_DEPRECATED, so "it compiles against 4.0" was an inference rather than something CI checked. It was also wrong: 4.0 constified four X509 name accessors and the build failed with -Werror until the previous commit. That is exactly the class of breakage this leg is meant to catch early, while it is still a deprecation and not yet a removal. No distribution ships OpenSSL 4.0 -- Ubuntu 26.04 is on 3.5, Debian sid and Alpine edge on 3.6 -- so the leg builds it from source and caches it, reusing the shape of the existing OpenSSL 3.6 slot: its own prefix under /opt so the system libraries sudo depends on are never overwritten, LD_LIBRARY_PATH scoped to the configure step for the auto/feature probes, and -Wl,-rpath for everything after. Kept cheap on purpose: configure, build, build the C tests, run them. No language modules and no pytest run, because nothing above the TLS layer depends on the OpenSSL version and the matrix above already covers it. The version check asserts rather than prints. If the include and library flags ever stop taking effect, configure silently falls back to the runner's system OpenSSL 3.x, and the leg would go green while testing nothing it exists for. --- .github/workflows/build-test.yml | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 701234107..5e3e00c89 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -28,6 +28,10 @@ env: # patch level lives here; the 3.6 "slot" (/opt/openssl-3.6, OPENSSL36_* vars) # is stable across patch bumps and intentionally not templated. OPENSSL_VERSION: 3.6.2 + # OpenSSL 4.0 for the openssl4 leg below. No distribution ships 4.0 yet, so + # like the 3.6 slot it is built from source; the /opt/openssl-4.0 slot is + # stable across patch bumps. + OPENSSL4_VERSION: 4.0.2 jobs: # Single source of truth for the test matrix. The version-tested language @@ -534,3 +538,80 @@ jobs: - name: Run libunit port_recv fd-ownership test run: ./build/unit_port_recv_test + + # OpenSSL 4.0 compile gate. 4.0 removed nothing Unit uses, but it deprecated + # more of what Unit still calls, and 5.0 is where the deprecated symbols + # actually go away -- so the value here is having a leg that fails when a new + # deprecation lands, instead of noticing at removal time. Deliberately cheap: + # a build plus the C test suite, no language modules and no pytest run, since + # nothing above the TLS layer changes with the OpenSSL version. + openssl4: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libpcre2-dev + + - name: Cache OpenSSL 4.0 + id: cache-openssl4 + uses: actions/cache@v6 + with: + path: /opt/openssl-4.0 + key: openssl-${{ env.OPENSSL4_VERSION }}-${{ runner.os }}-${{ runner.arch }} + + - name: Build OpenSSL 4.0 + if: steps.cache-openssl4.outputs.cache-hit != 'true' + run: | + cd /tmp + wget -q "https://www.openssl.org/source/openssl-${OPENSSL4_VERSION}.tar.gz" + tar -xzf "openssl-${OPENSSL4_VERSION}.tar.gz" + cd "openssl-${OPENSSL4_VERSION}" + ./Configure --prefix=/opt/openssl-4.0 shared no-docs + make -j$(nproc) + sudo mkdir -p /opt/openssl-4.0 + sudo make install_sw + + - name: Set OpenSSL 4.0 build environment + # Same isolation as the 3.6 slot: never overwrite the system libraries, + # because sudo links against them. + run: | + OSSL=/opt/openssl-4.0 + LIBDIR=$( [ -d "$OSSL/lib64" ] && echo "$OSSL/lib64" || echo "$OSSL/lib" ) + echo "OPENSSL4_INCDIR=${OSSL}/include" >> $GITHUB_ENV + echo "OPENSSL4_LIBDIR=${LIBDIR}" >> $GITHUB_ENV + + - name: Configure unit (--openssl --tests) + # LD_LIBRARY_PATH is scoped to this step only so that auto/feature test + # binaries load OpenSSL 4.0; exporting it globally would break sudo. + # The build itself needs no such help -- -Wl,-rpath below is baked into + # the binaries, and is honoured under sudo too. + env: + LD_LIBRARY_PATH: ${{ env.OPENSSL4_LIBDIR }} + run: | + ./configure \ + --tests \ + --openssl \ + --cc-opt="-I${OPENSSL4_INCDIR} -DOPENSSL_NO_DEPRECATED" \ + --ld-opt="-L${OPENSSL4_LIBDIR} -Wl,-rpath,${OPENSSL4_LIBDIR}" + + # Assert rather than print: if the include/lib flags ever stop taking + # effect, configure falls back to the runner's system OpenSSL 3.x and the + # leg would pass while testing nothing it was added for. + - name: Assert configure found OpenSSL 4.0 + run: | + grep NXT_HAVE_OPENSSL_VERSION build/include/nxt_auto_config.h + grep -q '#define NXT_HAVE_OPENSSL_VERSION "OpenSSL 4\.' \ + build/include/nxt_auto_config.h + + - name: Build unit + run: make -j4 + + - name: Build C tests + run: make -j4 tests + + - name: Run C tests + run: sudo ./build/tests From b5e0f37b6f7871d6b8accdf606a251ba8aecbf3d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 20:07:55 +0200 Subject: [PATCH 046/391] tls: use the OpenSSL 4.0 alert reason-code names OpenSSL 4.0 renamed the SSLv3-era alert reason codes to SSL_R_TLS_ALERT_* and removed the old spellings, so nxt_openssl_log_error_level() does not compile against it once deprecated APIs are disabled: eleven case labels name symbols that no longer exist. Nothing caught this before now because 3.x still carries the old names as deprecated aliases, so every build and review against a 3.x library sees them resolve. Define the new spelling in terms of the old one where the new one is missing, and label the cases with the new spelling. 1.1.1 and 3.x take the fallback; 4.0 uses its own definitions. The SSL_R_TLSV1_ALERT_* codes are deliberately left alone. They look like they belong to the same family, but 4.0 kept those names and defines no SSL_R_TLS_ALERT_ spelling for them -- bridging them would invent aliases for a rename that did not happen. Checked name by name against 4.0.2's sslerr.h rather than assumed from the pattern. Co-Authored-By: Claude Opus 5 --- src/nxt_openssl.c | 69 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/src/nxt_openssl.c b/src/nxt_openssl.c index 5589eccd5..bf2630d0c 100644 --- a/src/nxt_openssl.c +++ b/src/nxt_openssl.c @@ -1671,6 +1671,53 @@ nxt_openssl_conn_error(nxt_task_t *task, nxt_err_t err, const char *fmt, ...) } +/* + * OpenSSL 4.0 renamed the SSLv3-era alert reason codes to SSL_R_TLS_ALERT_* + * and removed the old spellings, so a build against it with deprecated APIs + * disabled cannot see SSL_R_SSLV3_ALERT_*. 1.1.1 and 3.x have only the old + * names. Define the new spelling in terms of the old one where it is + * missing, so the switch below needs one spelling rather than a version + * conditional per label. + * + * The SSL_R_TLSV1_ALERT_* codes are deliberately not bridged: 4.0 kept those + * names, and no SSL_R_TLS_ALERT_ spelling exists for them. + */ + +#ifndef SSL_R_TLS_ALERT_UNEXPECTED_MESSAGE +#define SSL_R_TLS_ALERT_UNEXPECTED_MESSAGE SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE +#endif +#ifndef SSL_R_TLS_ALERT_BAD_RECORD_MAC +#define SSL_R_TLS_ALERT_BAD_RECORD_MAC SSL_R_SSLV3_ALERT_BAD_RECORD_MAC +#endif +#ifndef SSL_R_TLS_ALERT_DECOMPRESSION_FAILURE +#define SSL_R_TLS_ALERT_DECOMPRESSION_FAILURE SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE +#endif +#ifndef SSL_R_TLS_ALERT_HANDSHAKE_FAILURE +#define SSL_R_TLS_ALERT_HANDSHAKE_FAILURE SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE +#endif +#ifndef SSL_R_TLS_ALERT_ILLEGAL_PARAMETER +#define SSL_R_TLS_ALERT_ILLEGAL_PARAMETER SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER +#endif +#ifndef SSL_R_TLS_ALERT_NO_CERTIFICATE +#define SSL_R_TLS_ALERT_NO_CERTIFICATE SSL_R_SSLV3_ALERT_NO_CERTIFICATE +#endif +#ifndef SSL_R_TLS_ALERT_BAD_CERTIFICATE +#define SSL_R_TLS_ALERT_BAD_CERTIFICATE SSL_R_SSLV3_ALERT_BAD_CERTIFICATE +#endif +#ifndef SSL_R_TLS_ALERT_UNSUPPORTED_CERTIFICATE +#define SSL_R_TLS_ALERT_UNSUPPORTED_CERTIFICATE SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE +#endif +#ifndef SSL_R_TLS_ALERT_CERTIFICATE_REVOKED +#define SSL_R_TLS_ALERT_CERTIFICATE_REVOKED SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED +#endif +#ifndef SSL_R_TLS_ALERT_CERTIFICATE_EXPIRED +#define SSL_R_TLS_ALERT_CERTIFICATE_EXPIRED SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED +#endif +#ifndef SSL_R_TLS_ALERT_CERTIFICATE_UNKNOWN +#define SSL_R_TLS_ALERT_CERTIFICATE_UNKNOWN SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN +#endif + + static nxt_uint_t nxt_openssl_log_error_level(nxt_err_t err) { @@ -1728,21 +1775,21 @@ nxt_openssl_log_error_level(nxt_err_t err) case SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING: /* 345 */ #endif case 1000:/* SSL_R_SSLV3_ALERT_CLOSE_NOTIFY */ - case SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE: /* 1010 */ - case SSL_R_SSLV3_ALERT_BAD_RECORD_MAC: /* 1020 */ + case SSL_R_TLS_ALERT_UNEXPECTED_MESSAGE: /* 1010 */ + case SSL_R_TLS_ALERT_BAD_RECORD_MAC: /* 1020 */ case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED: /* 1021 */ case SSL_R_TLSV1_ALERT_RECORD_OVERFLOW: /* 1022 */ - case SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE: /* 1030 */ - case SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE: /* 1040 */ - case SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER: /* 1047 */ + case SSL_R_TLS_ALERT_DECOMPRESSION_FAILURE: /* 1030 */ + case SSL_R_TLS_ALERT_HANDSHAKE_FAILURE: /* 1040 */ + case SSL_R_TLS_ALERT_ILLEGAL_PARAMETER: /* 1047 */ break; - case SSL_R_SSLV3_ALERT_NO_CERTIFICATE: /* 1041 */ - case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: /* 1042 */ - case SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE: /* 1043 */ - case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED: /* 1044 */ - case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED: /* 1045 */ - case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: /* 1046 */ + case SSL_R_TLS_ALERT_NO_CERTIFICATE: /* 1041 */ + case SSL_R_TLS_ALERT_BAD_CERTIFICATE: /* 1042 */ + case SSL_R_TLS_ALERT_UNSUPPORTED_CERTIFICATE: /* 1043 */ + case SSL_R_TLS_ALERT_CERTIFICATE_REVOKED: /* 1044 */ + case SSL_R_TLS_ALERT_CERTIFICATE_EXPIRED: /* 1045 */ + case SSL_R_TLS_ALERT_CERTIFICATE_UNKNOWN: /* 1046 */ case SSL_R_TLSV1_ALERT_UNKNOWN_CA: /* 1048 */ case SSL_R_TLSV1_ALERT_ACCESS_DENIED: /* 1049 */ case SSL_R_TLSV1_ALERT_DECODE_ERROR: /* 1050 */ From 6a6c5db19243cae275a68132a81b6bd5697fa325 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 19:36:02 +0200 Subject: [PATCH 047/391] data(eol): update java runtimes and record dependency floors (OpenSSL 1.1.1, Tomcat 9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Java date audit: all four .runtimes.java eol dates already match upstream. The validator resolves java against endoflife.date's *eclipse-temurin* product (pkg/eol/src/main.rs:162), not /api/java.json (which now 404s), and that API gives 17 -> 2027-10-31, 21 -> 2029-12-31, 25 -> 2031-09-30, 26 -> 2026-09-15 — i.e. exactly the eol values in the file. So no date changes; `--ci` reports zero java items before and after. Source: https://endoflife.date/api/eclipse-temurin.json Java 27 is not added: the file's convention (08644bb3) is to add a cycle after GA, and 27 is not yet in the API — `--new` flags only go 1.27 and perl 5.44, no java. Java 26 goes upstream-EOL 2026-09-15; its supported_until 2027-09 keeps it in the matrix, and it will pick up the "(EOL)" note on the next pass after that date. The only java edit is consistency: "lts": true added to 17 and 21, which the API marks lts and which EOL.md already labels LTS, matching the 25 entry. No consumer reads the flag (build-test.yml, release-docker.yml, pkg/docker/Makefile, pkg/docker/build-local.sh key off .version/.slim only). Also fixes the one pre-existing validator error, which is unrelated to java but blocks the eol-check PR gate on any change to this file: centos_stream 10 was recorded as EOL 2030-01 while upstream says 2030-05-31, so eol becomes 2030-05 and supported_until 2033-05 (EOL + 36mo OS grace), with the matching EOL.md row. Source: https://endoflife.date/api/centos-stream.json With that, `--ci` goes from exit 1 (errors: 1) to exit 0 (errors: 0, warnings: 14 — all "plan migration" notices, unchanged). New top-level "dependencies" key (plus "_comment_dependencies") records build/bundle floors that are not matrix variants — deliberately NOT under .runtimes, which would conjure Docker variants in all four consumers. It covers every library the build pulls in, i.e. every jar array in auto/modules/java, not just the headline ones: * openssl — floor 1.1.1 (upstream EOL 2023-09), declared by the auto/ssltls probe (PR #224); kept despite being upstream-EOL because it is what RHEL 8, Amazon Linux 2 (openssl11-devel, pkg/rpm/Makefile) and Debian 11 ship, and all three are inside the OS support window above. Two tested ceilings: 3.6.2 (OPENSSL_VERSION, /opt/openssl-3.6) and 4.0.2 (OPENSSL4_VERSION, /opt/openssl-4.0, the openssl4 job, built with -DOPENSSL_NO_DEPRECATED so any deprecated-API use fails). The 3.6 pin is an action item: it goes upstream-EOL 2026-11-01 and needs a bump to a supported branch (3.5 is the LTS, EOL 2030-04). Source: https://endoflife.date/api/openssl.json * tomcat — 9.0.x jars bundled by auto/modules/java, EOL "no earlier than" 2027-03-31. Source: https://tomcat.apache.org/whichversion.html * jetty — jetty-util / jetty-server / jetty-http 9.4.58.v20250814. ALREADY EOL: Eclipse ended community support for 9.4 on 2025-08-14 and the bundled build is that line's final release. Recorded with no supported_until — the grace policy covers matrix variants, not vendored jars — so it reads as an upgrade-or-replace decision (10/11 are EOL too; 12.x is the supported line). Source: https://endoflife.date/api/eclipse-jetty.json * ecj — Eclipse JDT batch compiler 3.26.0 (JSP compilation). eol null: endoflife.date has no product for it (/api/all.json lists only eclipse-jetty and eclipse-temurin under Eclipse), so there is no upstream date to validate against. Pinned build is from 2021-06-14 against a current 3.42.0 (2025-06-07) — stale, not expired. * classgraph — no floor, no upstream EOL schedule; pinned and bumped. pkg/eol/src/main.rs reads only _grace_*, .os and .runtimes, so the new key is inert for the validator. The four consumer derivations were run against the old and new file and diff clean, as does `make -C pkg/docker -n dockerfiles`. EOL.md gains a "Dependency Support" section mirroring the key, and a Rules bullet stating the grace policy does not apply to dependencies. Co-Authored-By: Claude Opus --- EOL.md | 25 ++++++++++++++++++++++++- pkg/eol.json | 46 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/EOL.md b/EOL.md index d7f8d63f7..b211387fb 100644 --- a/EOL.md +++ b/EOL.md @@ -45,7 +45,7 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Fedora | 43 | 2026-12 | 2029-12 | 3.14 | | Fedora | 44 | 2027-06 | 2030-06 | 3.14 | | CentOS Stream | 9 | 2027-05 | 2030-05 | 3.11 | -| CentOS Stream | 10 | 2030-01 | 2033-01 | 3.13 | +| CentOS Stream | 10 | 2030-05 | 2033-05 | 3.13 | | Amazon Linux | 2 | 2026-06 | 2029-06 | 3.7 ‡ | | Amazon Linux | 2023 | 2029-06 | 2032-06 | 3.11 | | Ubuntu (LTS) | 22.04 | 2027-04 | 2030-04 | 3.10 | @@ -66,6 +66,26 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). † Past upstream EOL; in FreeUnit grace period. ‡ Default Python shipped by this OS is itself past upstream EOL. FreeUnit does not backport fixes to that Python version. +## Dependency Support + +Build-time and bundled libraries. These are **not** matrix variants — nothing in +Docker or CI derives an image from this table — so the 1-year grace rule does not +apply to them. Machine-readable copy: the `dependencies` key in `pkg/eol.json`. + +† Bundled software that is already past upstream EOL. It carries no FreeUnit "drops" +date, because the grace policy applies to matrix variants, not to vendored jars; it +needs an upgrade or replacement decision instead. + +| Dependency | Role | Floor / Version | Upstream EOL | Notes | +|------------|------|-----------------|--------------|-------| +| OpenSSL | TLS backend (`--openssl`) | 1.1.1 (floor) | Sep 2023 | Kept because RHEL 8, Amazon Linux 2 (`openssl11-devel`) and Debian 11 ship it. Floor declared by the `auto/ssltls` probe (PR #224). | +| OpenSSL | tested ceiling | 3.6.2 (CI build) | Nov 2026 | `build-test.yml` builds it into `/opt/openssl-3.6` (`OPENSSL_VERSION`). **The pin needs a bump before 2026-11-01** (3.5 is the LTS, EOL Apr 2030). | +| OpenSSL | tested ceiling | 4.0.2 (CI build) | May 2027 | The `openssl4` job builds it into `/opt/openssl-4.0` (`OPENSSL4_VERSION`) and compiles with `-DOPENSSL_NO_DEPRECATED`, so any use of an API 4.0 deprecates fails the build. | +| Apache Tomcat | Servlet/JSP/EL API + Jasper jars bundled by the Java module | 9.0.x | Mar 2027 ([no earlier than](https://tomcat.apache.org/whichversion.html)) | Bundled by `auto/modules/java`; independent of the JDK variant. | +| Eclipse Jetty | `jetty-util` / `jetty-server` / `jetty-http` jars bundled by the Java module | 9.4.58.v20250814 | **Aug 2025 (EOL)** † | Jetty 9.4 lost community support on 2025-08-14 and the bundled build is its last release. Needs an upgrade-or-replace decision — Jetty 10/11 are EOL too; 12.x is the supported line. | +| Eclipse ECJ (JDT batch compiler) | JSP compilation jar bundled by the Java module | 3.26.0 | none published | Tracks Eclipse releases; endoflife.date has no product for it. Pinned build is from Jun 2021; current is 3.42.0 (Jun 2025). | +| ClassGraph | classpath scanning jar bundled by the Java module | latest | — | No upstream EOL schedule; pinned and bumped as needed. | + ## Rules - **Adding a runtime version:** when new upstream release reaches stable, FreeUnit adds @@ -77,6 +97,9 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). - **Security-only mode:** versions within 6 months of the FreeUnit drop date receive security fixes only — no new features backported. - **LTS runtimes (Java 17/21/25):** follow the upstream LTS schedule strictly. +- **Dependencies:** the floors in the Dependency Support section are not part of the + EOL + grace policy; they track what FreeUnit builds and bundles, not what it ships + as a variant. - **LTS OS (Ubuntu, RHEL, Debian):** 3-year extension applies to standard EOL, not extended security maintenance dates. diff --git a/pkg/eol.json b/pkg/eol.json index 97ccd6db3..757979a63 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -15,7 +15,7 @@ ], "centos_stream": [ {"version": "9", "eol": "2027-05", "supported_until": "2030-05", "default_python": "3.11"}, - {"version": "10", "eol": "2030-01", "supported_until": "2033-01", "default_python": "3.13"} + {"version": "10", "eol": "2030-05", "supported_until": "2033-05", "default_python": "3.13"} ], "amazonlinux": [ {"version": "2", "eol": "2026-06", "supported_until": "2029-06", "default_python": "3.7", "default_python_note": "OS-shipped; Python 3.7 upstream EOL Jun 2023"}, @@ -50,8 +50,8 @@ {"version": "1.26", "eol": "2027-02", "supported_until": "2028-02"} ], "java": [ - {"version": "17", "eol": "2027-10", "supported_until": "2028-10", "base": "eclipse-temurin"}, - {"version": "21", "eol": "2029-12", "supported_until": "2030-12", "base": "eclipse-temurin"}, + {"version": "17", "eol": "2027-10", "supported_until": "2028-10", "base": "eclipse-temurin", "lts": true}, + {"version": "21", "eol": "2029-12", "supported_until": "2030-12", "base": "eclipse-temurin", "lts": true}, {"version": "25", "eol": "2031-09", "supported_until": "2032-09", "base": "eclipse-temurin", "lts": true}, {"version": "26", "eol": "2026-09", "supported_until": "2027-09", "base": "eclipse-temurin"} ], @@ -87,5 +87,45 @@ "minimal": [ {"version": null, "eol": null, "supported_until": null} ] + }, + "_comment_dependencies": "Build-time and bundled library floors/ceilings. NOT a matrix: only .os and .runtimes feed the Docker/CI variant lists (build-test.yml, release-docker.yml, pkg/docker/Makefile, pkg/docker/build-local.sh), and pkg/eol/src/main.rs reads only _grace_*, .os and .runtimes — unknown top-level keys are ignored. Entries here are documentation for humans; the EOL + grace policy does not apply to them, so they carry no supported_until.", + "dependencies": { + "openssl": { + "role": "TLS backend (./configure --openssl)", + "floor": "1.1.1", + "floor_eol": "2023-09", + "floor_note": "Upstream-EOL but still supported by FreeUnit because it is what RHEL 8, Amazon Linux 2 (openssl11-devel, see pkg/rpm/Makefile and pkg/rpm/unit.spec.in) and Debian 11 ship, and all three are inside the OS support window above.", + "floor_declared_by": "auto/ssltls probe (PR #224), which also drops the pre-1.1.0 compatibility code in src/nxt_openssl.c", + "tested_ceiling": "3.6.2 and 4.0.2", + "tested_ceiling_note": "build-test.yml tests two ceilings from source: 3.6.2 (OPENSSL_VERSION env, built into /opt/openssl-3.6) and 4.0.2 (OPENSSL4_VERSION env, /opt/openssl-4.0, built by the openssl4 job with --cc-opt=-DOPENSSL_NO_DEPRECATED so any use of an API OpenSSL 4.0 deprecates fails the build). ACTION: OpenSSL 3.6 goes upstream-EOL 2026-11-01, so the OPENSSL_VERSION pin needs a bump to a supported branch (3.5 is the LTS, EOL 2030-04); 4.0 is supported to 2027-05.", + "source": "https://endoflife.date/api/openssl.json" + }, + "tomcat": { + "role": "Servlet/JSP/EL API + Jasper jars bundled by the java module (auto/modules/java, NXT_TOMCAT_VERSION)", + "version": "9.0.x", + "eol": "2027-03", + "eol_note": "Apache: \"End of support for Tomcat 9.0.x is expected no earlier than 31 March 2027\", with at least 12 months notice. Not a Docker variant — the jars are bundled into the java module regardless of JDK version.", + "source": "https://tomcat.apache.org/whichversion.html" + }, + "jetty": { + "role": "HTTP parsing/server jars bundled by the java module (auto/modules/java, NXT_JAR_VERSION under org/eclipse/jetty/): jetty-util, jetty-server, jetty-http", + "version": "9.4.58.v20250814", + "eol": "2025-08", + "eol_note": "ALREADY EOL. Eclipse ended community support for the Jetty 9.4 line on 2025-08-14, and 9.4.58.v20250814 — the exact build bundled here — is its final release (jetty.org/download.html lists it as (EOL); the 9.4 row reads \"2016-2025 ... EOL / Unsupported\"). No supported_until is recorded: the EOL + grace policy covers matrix variants, not bundled jars, and this needs an upgrade-or-replace decision (Jetty 10/11 are also EOL as of 2025-01-01; 12.0/12.1 are the supported lines and require Java 11+/17+), not a grace window.", + "source": "https://endoflife.date/api/eclipse-jetty.json" + }, + "ecj": { + "role": "Eclipse JDT batch compiler jar bundled by the java module (auto/modules/java, org/eclipse/jdt/), used to compile JSPs at runtime", + "version": "3.26.0", + "eol": null, + "eol_note": "Tracks Eclipse releases; no published EOL. endoflife.date has no product for it (checked /api/all.json — the only Eclipse products are eclipse-jetty and eclipse-temurin), so there is no upstream date to validate against. The pinned 3.26.0 is from 2021-06-14; the current release is 3.42.0 (2025-06-07), so this is stale rather than expired.", + "source": "https://central.sonatype.com/artifact/org.eclipse.jdt/ecj/versions" + }, + "classgraph": { + "role": "classpath scanning jar bundled by the java module (auto/modules/java, NXT_JAR_VERSION)", + "version": "latest", + "note": "No floor and no EOL schedule upstream: pinned to the current release and bumped as needed.", + "source": "https://github.com/classgraph/classgraph/releases" + } } } From 73eece042b258cd51cb2d34393410d08db8bd1b5 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 20:15:44 +0200 Subject: [PATCH 048/391] ci(build-test): configure python and java against the toolchain the matrix installed The "Configure python3" and "Configure java" steps run configure under sudo. sudo's secure_path replaces PATH, dropping the toolcache entry that actions/setup-python prepends, and pam_env reloads the system JAVA_HOME. So configure resolved the *system* interpreter and JDK, not the one the matrix leg asked for. Evidence, run 33195245849, job 98931628898 titled "test (python-3.14, ubuntu-latest)": Run actions/setup-python@v7 Successfully set up CPython (3.14.7) Configure python3 pythonLocation: /opt/hostedtoolcache/Python/3.14.7/x64 Configure python3 checking for Python version ... 3.12.3 and job 98931628841, "test (java-26, ubuntu-latest)": Configure java JAVA_HOME: /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/26.0.2-101/x64 Configure java using java.home /usr/lib/jvm/temurin-17-jdk-amd64 Affected legs: python-3.13 and python-3.14 both built against the runner's /usr/bin python 3.12.3; java-21, java-25 and java-26 all built against temurin-17. python-3.12 and java-17 happened to match the system versions, so five of the seven legs were testing a duplicate of an existing one. Pass the paths explicitly instead. Both variables are present in the step environment (the log lines above are the runner echoing them), and the shell expands them before sudo resets the environment, so they arrive as argv. The make targets are unchanged. auto/modules/python derives NXT_PYTHON_MODULE from basename ${NXT_PYTHON_CONFIG%-config*}, which is still "python3" for .../bin/python3-config; auto/modules/java defaults NXT_JAVA_MODULE to "java" regardless of --home. --lib-path only appends -L/-Wl,-rpath so the module can dlopen the toolcache libpython at test time. The go, node, php, ruby and perl legs configure without sudo and were already correct. The test harness needs the same treatment. test/unit/applications/lang/ java.py compiles the websocket test apps with a bare "javac", and the suite itself runs under "sudo -E pytest-3" -- -E preserves JAVA_HOME but not PATH, which secure_path still overrides. With configure now correct, the jars are built by the toolcache JDK while the test-time javac is still the system JDK 17, so it cannot read them: (run 33198523700, job 98942052049, java-21) Make java using java.home /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/21.0.12-1/x64 Run java tests stdout = b'.../websockets_mirror/app.java:4: error: cannot access OnMessage import j... 14 errors' E Failed: Can't run javac process. 28 failed, 51 passed, 40 skipped Prefer $JAVA_HOME/bin/javac when JAVA_HOME is set and that file exists, and fall back to a bare "javac" otherwise, so the suite still works outside CI. Resolve JAVA_HOME before handing it to configure. On ubuntu-latest the toolcache entry for a preinstalled JDK is a symlink into /usr/lib/jvm, and auto/modules/java derives its two JDK paths from different places: NXT_JAVA_HOME is the --home argument as given, while NXT_JAVA_LIB_PATH comes from "$NXT_JAVA -XshowSettings", which reports the *resolved* location. The two then disagree: (run 33203354611, job 98947191768, java-17) using java.home /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk/17.0.20-1/x64 checking library path ... /usr/lib/jvm/temurin-17-jdk-amd64/lib nxt_java_mounts[] binds NXT_JAVA_HOME into the rootfs, so under chroot the JVM got the symlink path bound but not the real one its own libjvm resolves against, and failed with "Error occurred during initialization of VM": test_java_isolation_rootfs.py::test_java_isolation_rootfs_chroot_war E AssertionError: war 1 failed, 78 passed, 40 skipped readlink -f makes both derivations agree. It is a no-op when JAVA_HOME is already a real directory, which is the case for JDKs setup-java downloads rather than adopts, and for a plain system JDK. test_python_isolation_rootfs_no_language_deps hardcoded the stdlib prefix. The module itself is fine: auto/modules/python builds nxt_python_mounts[] by running the *configured* interpreter and taking the sys.path entry whose basename is "pythonX.Y", so a toolcache build mounts the toolcache stdlib and the app starts. The log confirms it -- "enabled language_deps" passed, and only the mount-location assertion failed: (run 33207373983, job 98951986445, python-3.13) assert client.get()['status'] == 200, 'enabled language_deps' # passed > assert waitformount(python_path), 'language_deps mount' E AssertionError: language_deps mount E + where False = waitformount('/tmp/unit-test-gz4oma6l/usr') 1 failed, 699 passed, 401 skipped The bind mount was at /opt/hostedtoolcache/Python/3.13.15/x64/lib/ python3.13; the test only ever looked under /usr, which holds solely for a build against the system interpreter. ("Failed to import encodings module" in unit.log belongs to the first half of the test, which deliberately starts the app with language_deps disabled.) Match the stdlib directory instead of assuming its prefix, so the test covers venv, /usr/local and toolcache builds as well as the system one. Pin the embedded interpreter's home on the python legs. Unit does not tell the interpreter which prefix it was built against, so CPython falls back to resolving one by looking for "python3" on PATH. Under sudo that is /usr/bin/python3, and the prefix is accepted whenever /lib/pythonX.Y exists -- true only when the system interpreter's minor version matches. On python-3.12 that yields the toolcache libpython3.12 running the *system* stdlib, and Debian compiles _socket into libpython while vanilla CPython ships it in lib-dynload, so every socket-using app dies: (run 33201125801, job 98951986462, python-3.12; toolcache 3.12.14, system 3.12.3) [alert] [unit] Python failed to import module 'asyncio' File "/usr/lib/python3.12/socket.py", line 52, in import _socket ModuleNotFoundError: No module named '_socket' 64 failed, 638 passed, 399 skipped, 58 errors python-3.13 and 3.14 are unaffected: no /usr/lib/python3.13 or 3.14 exists, the PATH-derived prefix is rejected, and CPython falls back to its compiled-in toolcache prefix. PYTHONHOME cannot simply be exported by the workflow. unitd inherits the pytest process environment, so it would rebind pytest's own interpreter too, and PYTHONHOME drops the dist-packages directories that hold pytest itself: $ /usr/bin/python3 -c "import CommandNotFound" # OK $ PYTHONHOME= /usr/bin/python3 -c "import CommandNotFound" ModuleNotFoundError: No module named 'CommandNotFound' So the workflow exports UNIT_PYTHONHOME and conftest translates it to PYTHONHOME for the unitd child alone. Unset, the launch is byte-for-byte what it was. pythonLocation is not stable across the job. A second setup-python runs before the test step to provide pytest, with python-version '3', and overwrites pythonLocation with the newest 3.x in the toolcache. Reading it in the test step would hand conftest a 3.14 prefix on the 3.12 and 3.13 legs. Record UNIT_PYTHON_PREFIX right after the matrix setup-python instead, and use it for both the configure arguments and UNIT_PYTHONHOME. Configure runs before the second setup-python and so was reading the right value already; it uses the recorded prefix now for the same reason. This is a workaround for a product gap, not a fix: the module derives its rootfs mount list from the configured interpreter but never pins the runtime interpreter to the same prefix. Setting PyConfig.home from the configure-time sys.base_prefix belongs in src/python/nxt_python.c and is filed separately. No other language helper shells out to a matrix-installed toolchain binary; go.py runs "which go", and the go legs resolve it fine under sudo. Regression introduced by 8032ce31 (2024-02-27). Co-Authored-By: Claude Opus --- .github/workflows/build-test.yml | 19 +++++++++++-- test/conftest.py | 17 +++++++++++- test/test_python_isolation.py | 42 ++++++++++++++++++++++------- test/unit/applications/lang/java.py | 13 ++++++++- 4 files changed, 78 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 5e3e00c89..facc2c403 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -271,7 +271,7 @@ jobs: - name: Configure java run: | - sudo ./configure java + sudo ./configure java --home="$(readlink -f "${JAVA_HOME}")" if: steps.metadata.outputs.module == 'java' - name: Make java @@ -357,9 +357,19 @@ jobs: python-version: '${{ steps.metadata.outputs.version }}' if: steps.metadata.outputs.module == 'python' + # A second setup-python below installs python-version '3' for pytest and + # overwrites pythonLocation with whatever the newest 3.x is. Record the + # matrix interpreter's prefix while it is still the current one. + - name: Record python prefix + run: | + echo "UNIT_PYTHON_PREFIX=${pythonLocation}" >> "$GITHUB_ENV" + if: steps.metadata.outputs.module == 'python' + - name: Configure python3 run: | - sudo ./configure python --config=python3-config + sudo ./configure python \ + --config="${UNIT_PYTHON_PREFIX}/bin/python3-config" \ + --lib-path="${UNIT_PYTHON_PREFIX}/lib" if: steps.metadata.outputs.module == 'python' - name: Make python3 @@ -501,6 +511,11 @@ jobs: - name: Run ${{ steps.metadata.outputs.module }} tests run: | + # Pin unitd's embedded interpreter to the toolcache prefix it was + # built against; conftest turns this into PYTHONHOME for unitd only. + if [ "${{ steps.metadata.outputs.module }}" == "python" ]; then + export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" + fi if [ "${{ matrix.build }}" == "wasm-wasi-component" ]; then pytest-3 --print-log ${{ steps.metadata.outputs.testpath }} else diff --git a/test/conftest.py b/test/conftest.py index 77e60b050..4cf7f8a17 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -521,7 +521,22 @@ def unit_run(state_dir=None): # router, app workers) shares one group we can signal as a unit — and # which can never be confused with the pytest runner's own group or an # unrelated unitd already running on the box. - p = subprocess.Popen(unitd_args, stderr=log, start_new_session=True) + # UNIT_PYTHONHOME pins the embedded interpreter's stdlib to the + # prefix unit's python module was built against. It cannot be passed + # as PYTHONHOME from outside: unitd inherits this process's + # environment, and PYTHONHOME here would also rebind pytest's own + # interpreter, dropping dist-packages (where pytest itself lives) from + # its sys.path. Needed when unit is built against a non-system + # interpreter whose minor version matches the system one, because + # CPython then resolves its prefix by finding "python3" on PATH. + unitd_env = os.environ.copy() + pythonhome = unitd_env.pop('UNIT_PYTHONHOME', None) + if pythonhome: + unitd_env['PYTHONHOME'] = pythonhome + + p = subprocess.Popen( + unitd_args, stderr=log, start_new_session=True, env=unitd_env + ) unit_instance['process'] = p # Record the group id (== leader pid) immediately, on disk and in-memory, diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index 4e27282a2..26679e6c4 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -8,8 +8,6 @@ from unit.applications.lang.python import ApplicationPython from unit.option import option from unit.utils import findmnt -from unit.utils import waitformount -from unit.utils import waitforunmount prerequisites = {'modules': {'python': 'any'}, 'features': {'isolation': True}} @@ -78,29 +76,55 @@ def test_python_isolation_rootfs(is_su, require, temp_dir): assert ret['body']['FileExists'], 'application exists in rootfs' +def language_deps_mounts(temp_dir): + # The stdlib bind mount lands at , and auto/modules/ + # python picks that directory as the sys.path entry whose basename is + # "pythonX.Y". It is /usr/lib/pythonX.Y only when unit was built against + # the system interpreter; a toolcache, venv or /usr/local build mounts a + # different prefix, so match the stdlib directory rather than assuming one. + return [ + line + for line in findmnt().splitlines() + if line.startswith(f'{temp_dir}/') + and re.search(r'/python\d+\.\d+', line.split(' ')[0]) + ] + + +def waitfor(predicate, timeout=50): + for _ in range(timeout): + if predicate(): + return True + + time.sleep(0.1) + + return False + + def test_python_isolation_rootfs_no_language_deps(require, temp_dir): require({'privileged_user': True}) isolation = {'rootfs': temp_dir, 'automount': {'language_deps': False}} client.load('empty', isolation=isolation) - python_path = f'{temp_dir}/usr' - - assert findmnt().find(python_path) == -1 + assert not language_deps_mounts(temp_dir) assert client.get()['status'] != 200, 'disabled language_deps' - assert findmnt().find(python_path) == -1 + assert not language_deps_mounts(temp_dir) isolation['automount']['language_deps'] = True client.load('empty', isolation=isolation) - assert findmnt().find(python_path) == -1 + assert not language_deps_mounts(temp_dir) assert client.get()['status'] == 200, 'enabled language_deps' - assert waitformount(python_path), 'language_deps mount' + assert waitfor( + lambda: bool(language_deps_mounts(temp_dir)) + ), 'language_deps mount' client.conf({"listeners": {}, "applications": {}}) - assert waitforunmount(python_path), 'language_deps unmount' + assert waitfor( + lambda: not language_deps_mounts(temp_dir) + ), 'language_deps unmount' def test_python_isolation_procfs(require, temp_dir): diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index 40fda07e0..25258c01d 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -63,8 +63,19 @@ def prepare_env(self, script): if not ws_jars: pytest.fail('websocket api jar not found.') + # The suite runs under sudo, whose secure_path drops the + # toolcache directory that actions/setup-java prepends, so a bare + # "javac" resolves to the system JDK rather than the one this + # matrix leg installed and built the jars with. + javac_bin = 'javac' + java_home = os.environ.get('JAVA_HOME') + if java_home: + javac_home = os.path.join(java_home, 'bin', 'javac') + if os.path.isfile(javac_home): + javac_bin = javac_home + javac = [ - 'javac', + javac_bin, '-target', '8', '-source', From 446af3a52af7eb535f8ee08e439cb9f08a33b915 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 03:20:38 +0200 Subject: [PATCH 049/391] ci(dependabot): target the default branch, not pre-1.35.6 Dependabot has been pointed at pre-1.35.6, a branch that stopped being the integration target long ago. Two consequences, the second worse than the first. Version updates branch from pre-1.35.6, so their diffs carry commits master already has. #240 arrived with three of them -- setup-python 6->7, setup-go 6->7 and setup-node 4->7, all present on master under different shas -- around the one bump it was for, and had to be rebuilt before it could be merged. Security updates were not being raised at all. Dependabot only ever opens those against the repository's default branch, and setting target-branch suppresses them for the ecosystem. So the weekly version bumps kept arriving while nothing reported a vulnerable action. Removing the key rather than writing "master" keeps this correct if the default branch is ever renamed. Co-Authored-By: Claude Opus 5 --- .github/dependabot.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2cfde526d..c75e875a0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,6 +2,5 @@ version: 2 updates: - package-ecosystem: github-actions directory: "/" - target-branch: "pre-1.35.6" schedule: interval: weekly From b0d08ede4e0acadabe559faa04f927ffc7bb52aa Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 04:14:18 +0200 Subject: [PATCH 050/391] docs(eol): flag Go 1.25, Perl 5.38 and Debian 12 as past upstream EOL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three rows sat past their upstream EOL date without the "(EOL) †" marker that every other such row carries, so the tables read as if the versions were still upstream-supported. Dates confirmed against endoflife.date rather than inferred from the file: Go 1.25 2026-08-19 Perl 5.38 2026-07-02 Debian 12 2026-07-11 (LTS to 2028-06-30) All three stay in the matrix: FreeUnit's own three-year grace window carries them to 2027-08, 2027-07 and 2029-07 respectively. The marker records that upstream stopped, not that FreeUnit has. Go 1.25 is the one worth noticing. It went EOL ten days ago and the checker has not reported it, because eol.json stores month granularity ("2026-08") and the comparison only fires once the month is fully past -- it would have surfaced in September. CI still builds go-1.25 and pkg/docker/Dockerfile.go-1.25 still ships it; that is consistent with the grace window, but it should be a deliberate choice rather than an unflagged one. Checker output: 14 warnings before, 13 after, 0 errors either way. Only Perl 5.38 produced an "add (EOL) flag" warning, which is now gone -- the Debian and Go rows do not raise that class, for reasons the flag does not change: an OS past its date takes the "plan migration" path, and Go 1.25's month has not fully elapsed. Go 1.24 is deliberately left as it is: EOL.md keeps a marked row while pkg/eol.json has none, because the version is intentionally not built (.github/workflows/release-docker.yml:23) and eol.json tracks what is. Co-Authored-By: Claude Opus 5 --- EOL.md | 6 +++--- pkg/eol.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/EOL.md b/EOL.md index b211387fb..098fcbe01 100644 --- a/EOL.md +++ b/EOL.md @@ -11,7 +11,7 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Runtime | Version | Upstream EOL | FreeUnit Drops | |---------|---------|--------------|----------------| | Go | 1.24 (EOL) † | Feb 2026 | Feb 2027 | -| Go | 1.25 | Aug 2026 | Aug 2027 | +| Go | 1.25 (EOL) † | Aug 2026 | Aug 2027 | | Go | 1.26 | Feb 2027 | Feb 2028 | | Java (JSC) | 17 (LTS) | Oct 2027 | Oct 2028 | | Java (JSC) | 21 (LTS) | Dec 2029 | Dec 2030 | @@ -21,7 +21,7 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Node.js | 22 (LTS) | Apr 2027 | Apr 2028 | | Node.js | 24 (LTS) | Apr 2028 | Apr 2029 | | Node.js | 26 (LTS) | Apr 2029 | Apr 2030 | -| Perl | 5.38 | Jul 2026 | Jul 2027 | +| Perl | 5.38 (EOL) † | Jul 2026 | Jul 2027 | | Perl | 5.40 | Jun 2027 | Jun 2028 | | Perl | 5.42 | Jul 2028 | Jul 2029 | | PHP | 8.3 | Dec 2027 | Dec 2028 | @@ -52,7 +52,7 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Ubuntu (LTS) | 24.04 | 2029-05 | 2032-05 | 3.12 | | Ubuntu (LTS) | 26.04 | 2031-04 | 2034-04 | 3.13 | | Debian | 11 (bullseye) (EOL) † | 2024-08 | 2027-08 | 3.9 | -| Debian | 12 (bookworm) | 2026-07 | 2029-07 | 3.11 | +| Debian | 12 (bookworm) (EOL) † | 2026-07 | 2029-07 | 3.11 | | Debian | 13 (trixie) | 2028-08 | 2031-08 | 3.13 | | RHEL | 8 | 2029-05 | 2032-05 | 3.8 ‡ | | RHEL | 9 | 2032-05 | 2035-05 | 3.11 | diff --git a/pkg/eol.json b/pkg/eol.json index 757979a63..53bca3b01 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -28,7 +28,7 @@ ], "debian": [ {"version": "11", "eol": "2024-08", "supported_until": "2027-08", "default_python": "3.9", "codename": "bullseye", "note": "EOL"}, - {"version": "12", "eol": "2026-07", "supported_until": "2029-07", "default_python": "3.11", "codename": "bookworm"}, + {"version": "12", "eol": "2026-07", "supported_until": "2029-07", "default_python": "3.11", "codename": "bookworm", "note": "EOL"}, {"version": "13", "eol": "2028-08", "supported_until": "2031-08", "default_python": "3.13", "codename": "trixie"} ], "rhel": [ @@ -46,7 +46,7 @@ }, "runtimes": { "go": [ - {"version": "1.25", "eol": "2026-08", "supported_until": "2027-08"}, + {"version": "1.25", "eol": "2026-08", "supported_until": "2027-08", "note": "EOL"}, {"version": "1.26", "eol": "2027-02", "supported_until": "2028-02"} ], "java": [ @@ -62,7 +62,7 @@ {"version": "26", "eol": "2029-04", "supported_until": "2030-04", "lts": true} ], "perl": [ - {"version": "5.38", "eol": "2026-07", "supported_until": "2027-07"}, + {"version": "5.38", "eol": "2026-07", "supported_until": "2027-07", "note": "EOL"}, {"version": "5.40", "eol": "2027-06", "supported_until": "2028-06"}, {"version": "5.42", "eol": "2028-07", "supported_until": "2029-07"} ], From cd1efd933774cfeb20f7f208c7be7e49aa488846 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 20:40:16 +0200 Subject: [PATCH 051/391] docs(changes): complete the 1.36.1 entries Add the remaining 18 user-visible changes to the 1.36.1 changelog block and set its header date to the 28 Aug 2026 cut date. CHANGES is regenerated from docs/changes.xml (36 entries). Co-Authored-By: Claude Opus --- CHANGES | 167 +++++++++++++++++++++++++++++++- docs/changes.xml | 245 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 405 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index c1c1ab4e6..7f02dc982 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -Changes with FreeUnit 1.36.1 02 Aug 2026 +Changes with FreeUnit 1.36.1 28 Aug 2026 *) Feature: configuration validation errors now say where they are. A rejected configuration request carries an RFC 6901 JSON Pointer in @@ -12,6 +12,18 @@ Changes with FreeUnit 1.36.1 02 Aug 2026 the existing response fields and the error message wording are unchanged, and each new member is omitted where it does not apply. + *) Feature: the /status API reports OpenTelemetry span export health as + "telemetry": {"spans": {"exported": N, "failed": M}} -- the spans the + collector accepted, and the spans whose export failed. Both are + reported because a failure count alone cannot distinguish a healthy + pipeline from one that has never exported anything. The object is + omitted entirely unless Unit was built with OpenTelemetry support, + "/config/settings/telemetry" is set, and the exporter was built + successfully, so its absence is how a client tells telemetry is off + rather than idle. The counters are cumulative since the exporter was + last built, so changing any telemetry setting restarts them, while + re-applying an identical configuration does not. + *) Bugfix: validate the shared memory segment identifier and geometry an application process supplies in its NXT_PORT_MSG_MMAP message. The router checked only the pid fields, then used the peer-authored @@ -93,9 +105,10 @@ Changes with FreeUnit 1.36.1 02 Aug 2026 *) Change: recycle connection structures through a per-engine freelist, embed the static file context in the request, recycle static file buffer descriptors through a thread-local freelist, and resolve the - first eight routing variables and the first sixteen response header - fields from arrays embedded in the request. Each avoids or reuses an - allocation on a per-connection or per-request hot path. + first eight routing variables and the first sixteen request and + response header fields from arrays embedded in the request. Each + avoids or reuses an allocation on a per-connection or per-request hot + path. *) Change: the WebAssembly component module no longer supports guest-initiated outbound HTTP. Serving Unit never needed it, and the @@ -119,6 +132,152 @@ Changes with FreeUnit 1.36.1 02 Aug 2026 njs template strings express comparisons and regular expressions ("`${uri == '/admin'}`", "`${/^\/admin/.test(uri)}`"). + *) Change: build against OpenSSL 4.0, which constified the return types + of the X509 name accessors and deprecated + X509_NAME_get_text_by_NID(); SNI certificate matching and the + "certificates" control API now read the certificate name through + X509_NAME_get_index_by_NID() and the entry accessors instead, and a + new CI job builds and runs the C test suite against OpenSSL 4.0 with + deprecated APIs disabled, so it fails on any new use of a deprecated + function rather than only on a removal. + + *) Change: OpenSSL 1.1.1 or later is now required to build with + "--openssl"; configure fails with an explicit message on older + releases. Builds against OpenSSL 1.0.x have not worked since 1.35.2, + when the TLS module started to use OpenSSL_version_num(); the + unreachable pre-1.1.0 compatibility code is removed. + + *) Bugfix: close the engine port when a router worker thread exits. + Lowering settings/listen_threads deletes engines, and the thread exit + handler only dropped a port reference, and the runtime held another, + so nothing ever called nxt_port_close(): every deleted engine leaked + its socket pair and its queue descriptor, three per engine, for the + life of the router. The count only ever rose, so a configuration that + raises and lowers the thread count walked the router towards its + descriptor limit. + + *) Bugfix: close the descriptors a forged or malformed port message + leaves behind. A message carries up to two descriptors whatever its + type normally uses, and the dispatcher reclaims nothing once a + handler has returned. The main process kept the second descriptor on + the whoami path, the router lost both when a conf data message named + no reply port, and a PROCESS_READY naming an unknown process closed + neither. A peer that is already compromised could exhaust the + descriptor table of the receiving process -- the main process on the + whoami path -- by repeating any of them. + + *) Bugfix: bound the log file slot a CHANGE_FILE message claims, and + stop leaking the descriptor it carries. The slot index arrived from a + peer and was used to walk the runtime's log file list without a range + check, and a message whose payload was shorter than the index was + read past its end. The rejection paths now close what the message + attached, and nxt_file_redirect() releases the descriptor when dup2() + fails rather than only on success -- which also ends a leak of one + descriptor per unrotatable log file on every SIGUSR1. + + *) Bugfix: a NEW_PORT or PROCESS_READY message carrying a shared-memory + queue shorter than the queue type faulted the receiving process + (main, the prototype, or the router) with SIGBUS on first access; the + object size is now checked before mapping, a short queue is refused + with a warning, and the router no longer leaks the descriptor of a + queue it refuses. + + *) Bugfix: a NEW_PORT message announcing a port that already existed, or + one that could not be created, left an uninitialized stack value + behind; the controller dereferenced it unconditionally on every + NEW_PORT, and main and the router relied on a NULL check that could + not catch non-NULL garbage. The result is now cleared explicitly + whenever no port is created. + + *) Bugfix: close the file descriptors a NEW_PORT or MMAP message left + behind instead of leaking them. The discovery and application + prototype processes kept every port queue descriptor open for the + life of the process, the main process kept the queue descriptor of + any port it had not just created, the router lost it on ports it does + not map a queue for, the controller never closed it, and the mmap + handler leaked its second descriptor on every path and returned + closing nothing at all when the first was missing; a message type + with no handler also went uncollected. A NEW_PORT for a port that + already has a queue now refuses the new descriptor instead of mapping + it over the live queue and leaking the mapping it replaced, which had + let a repeated announcement re-point an established port at memory of + the sender's choosing. + + *) Bugfix: session tickets were silently disabled when Unit was built + against an OpenSSL 3.x library compiled with the deprecated 3.0 API + removed (OPENSSL_NO_DEPRECATED_3_0), because the feature guard tested + for a control macro that is itself declared only under that API and + so was never defined; the session ticket key callback now registers + through SSL_CTX_set_tlsext_ticket_key_evp_cb() and keys an EVP_MAC + context on OpenSSL 3.0 and later, independent of the deprecated + HMAC_CTX API. + + *) Bugfix: stop building OpenTelemetry span attributes for spans the + sampler has already dropped. set_attribute() on a non-recording span + discards its argument, but the argument was built first -- the + sprintf round-trips, the pool allocations and two owned strings per + attribute -- and nothing asked whether the span was recording. + Lowering sampling_ratio therefore bought back the exporter and + nothing on the request path. The sampler's verdict is now read once + when the span is created and the attribute stages consult it, which + removes most of telemetry's request-path cost at a low sampling + ratio. Context propagation is deliberately outside that check: the + traceparent must reach the peer and the application whatever the + sampling decision was. + + *) Bugfix: do not attach the traceparent to an OpenTelemetry span as a + "Unit Attribute" event. It duplicated the trace and span identifiers + the span context already carries, nothing consumed it, and it was + added twice on a request that took the error path with a trace it had + started itself, because the error path re-runs header propagation. + + *) Bugfix: OpenTelemetry spans were lost at the edges of a request's + life. A request that ended before its span was collected -- an error + path, a dropped connection -- left the span allocated and never + exported it; the span is now tied to the memory pool that holds it + and is exported with an error status when that pool is released, so + an abandoned request appears in the trace instead of vanishing. + + *) Bugfix: flush the OpenTelemetry span pipeline before the router + exits. Exit and reload discarded the pipeline with up to 4096 spans + still queued, losing every span produced since the last export. The + pipeline is now flushed with a 2 second bound, so a slow or + unreachable collector delays shutdown by at most that long rather + than either hanging it or costing the queued spans. + + *) Bugfix: rebuild the OpenTelemetry exporter only when the telemetry + settings have changed. Every configuration change rebuilt it, so + adding a route or changing an application stalled reconfiguration + against an unreachable collector even though it did not touch + "/config/settings/telemetry". The exporter is now rebuilt only when + the endpoint, protocol, sampling ratio or batch size actually differs + from the running one. + + *) Change: the OpenTelemetry span attributes "http.response.status_code" + and "http.request.body.size" are now emitted as OTLP integer values, + as the HTTP semantic conventions specify, instead of strings. A + collector or query that matches these attributes as strings must be + adapted; one that reads them as integers, or that relied on the + convention rather than on Unit's previous output, needs no change. + + *) Change: the work a traced request does on the request path is + reduced: attribute keys are no longer copied onto the heap per + request, the two attributes above no longer round-trip through + strings, and a span stage now crosses into the tracer once instead of + once per attribute. Together with the sampling fix above, the + telemetry overhead of a request measured 27% lower at + "sampling_ratio" 1.0 and 53% lower at 0.1. + + *) Change: the Java module now bundles Apache Tomcat 9.0.121 (from + 9.0.110) and ClassGraph 4.8.186 (from 4.8.181). The upstream Tomcat + 9.0.121 release fixes CVE-2026-66299 (unbounded buffer for + undelivered WebSocket messages, a memory exhaustion by a slow + client), CVE-2026-73180, CVE-2026-68569, CVE-2026-66422 and + CVE-2026-65927. FreeUnit bundles only the servlet, JSP and EL API + jars, Jasper and the Tomcat utility jars -- not catalina, the + WebSocket implementation or the example webapps -- so the bump keeps + the jars it ships on a supported release. + Changes with FreeUnit 1.36.0 16 Jul 2026 diff --git a/docs/changes.xml b/docs/changes.xml index c43fd4070..ab0b746c9 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -7,7 +7,7 @@ @@ -25,6 +25,21 @@ apply. + + +the /status API reports OpenTelemetry span export health as "telemetry": +{"spans": {"exported": N, "failed": M}} -- the spans the collector accepted, +and the spans whose export failed. Both are reported because a failure count +alone cannot distinguish a healthy pipeline from one that has never exported +anything. The object is omitted entirely unless Unit was built with +OpenTelemetry support, "/config/settings/telemetry" is set, and the exporter +was built successfully, so its absence is how a client tells telemetry is off +rather than idle. The counters are cumulative since the exporter was last +built, so changing any telemetry setting restarts them, while re-applying an +identical configuration does not. + + + validate the shared memory segment identifier and geometry an application @@ -155,8 +170,8 @@ and be accepted. recycle connection structures through a per-engine freelist, embed the static file context in the request, recycle static file buffer descriptors through a thread-local freelist, and resolve the first eight routing variables and the -first sixteen response header fields from arrays embedded in the request. -Each avoids or reuses an allocation on a per-connection or +first sixteen request and response header fields from arrays embedded in the +request. Each avoids or reuses an allocation on a per-connection or per-request hot path. @@ -193,6 +208,230 @@ comparisons and regular expressions + + +build against OpenSSL 4.0, which constified the return types of the X509 name +accessors and deprecated X509_NAME_get_text_by_NID(); SNI certificate matching +and the "certificates" control API now read the certificate name through +X509_NAME_get_index_by_NID() and the entry accessors instead, and a new CI job +builds and runs the C test suite against OpenSSL 4.0 with deprecated APIs +disabled, so it fails on any new use of a deprecated function rather than only +on a removal. + + + + + +OpenSSL 1.1.1 or later is now required to build with "--openssl"; configure +fails with an explicit message on older releases. Builds against OpenSSL 1.0.x +have not worked since 1.35.2, when the TLS module started to use +OpenSSL_version_num(); the unreachable pre-1.1.0 compatibility code is +removed. + + + + + +close the engine port when a router worker thread exits. Lowering +settings/listen_threads deletes engines, and the thread exit handler only +dropped a port reference, and the runtime held another, so nothing ever +called nxt_port_close(): every deleted engine leaked its socket pair and its +queue descriptor, three per engine, for the life of the router. The count +only ever rose, so a configuration that raises and lowers the thread count +walked the router towards its descriptor limit. + + + + + +close the descriptors a forged or malformed port message leaves behind. A +message carries up to two descriptors whatever its type normally uses, and +the dispatcher reclaims nothing once a handler has returned. The main +process kept the second descriptor on the whoami path, the router lost both +when a conf data message named no reply port, and a PROCESS_READY naming an +unknown process closed neither. A peer that is already compromised could +exhaust the descriptor table of the receiving process -- the main process on +the whoami path -- by repeating any of them. + + + + + +bound the log file slot a CHANGE_FILE message claims, and stop leaking the +descriptor it carries. The slot index arrived from a peer and was used to +walk the runtime's log file list without a range check, and a message whose +payload was shorter than the index was read past its end. The rejection +paths now close what the message attached, and nxt_file_redirect() releases +the descriptor when dup2() fails rather than only on success -- which also +ends a leak of one descriptor per unrotatable log file on every SIGUSR1. + + + + + +a NEW_PORT or PROCESS_READY message carrying a shared-memory queue shorter +than the queue type faulted the receiving process (main, the prototype, or the +router) with SIGBUS on first access; the object size is now checked before +mapping, a short queue is refused with a warning, and the router no longer +leaks the descriptor of a queue it refuses. + + + + + +a NEW_PORT message announcing a port that already existed, or one that could +not be created, left an uninitialized stack value behind; the controller +dereferenced it unconditionally on every NEW_PORT, and main and the router +relied on a NULL check that could not catch non-NULL garbage. The result is +now cleared explicitly whenever no port is created. + + + + + +close the file descriptors a NEW_PORT or MMAP message left behind instead of +leaking them. The discovery and application prototype processes kept every +port queue descriptor open for the life of the process, the main process kept +the queue descriptor of any port it had not just created, the router lost it +on ports it does not map a queue for, the controller never closed it, and the +mmap handler leaked its second descriptor on every path and returned closing +nothing at all when the first was missing; a message type with no handler also +went uncollected. A NEW_PORT for +a port that already has a queue now refuses the new descriptor instead of +mapping it over the live queue and leaking the mapping it replaced, which had +let a repeated announcement re-point an established port at memory of the +sender's choosing. + + + + + +session tickets were silently disabled when Unit was built against an OpenSSL +3.x library compiled with the deprecated 3.0 API removed +(OPENSSL_NO_DEPRECATED_3_0), because the feature guard tested for a control +macro that is itself declared only under that API and so was never defined; +the session ticket key callback now registers through +SSL_CTX_set_tlsext_ticket_key_evp_cb() and keys an EVP_MAC context on OpenSSL +3.0 and later, independent of the deprecated HMAC_CTX API. + + + + + +stop building OpenTelemetry span attributes for spans the sampler has +already dropped. set_attribute() on a non-recording span discards its +argument, but the argument was built first -- the sprintf round-trips, the +pool allocations and two owned strings per attribute -- and nothing asked +whether the span was recording. Lowering sampling_ratio therefore bought +back the exporter and nothing on the request path. The sampler's verdict is +now read once when the span is created and the attribute stages consult it, +which removes most of telemetry's request-path cost at a low sampling +ratio. Context propagation is deliberately outside that check: the +traceparent must reach the peer and the application whatever the sampling +decision was. + + + + + +do not attach the traceparent to an OpenTelemetry span as a "Unit Attribute" +event. It duplicated the trace and span identifiers the span context +already carries, nothing consumed it, and it was added twice on a request +that took the error path with a trace it had started itself, because the +error path re-runs header propagation. + + + + + +OpenTelemetry spans were lost at the edges of a request's life. A request +that ended before its span was collected -- an error path, a dropped +connection -- left the span allocated and never exported it; the span is now +tied to the memory pool that holds it and is exported with an error status +when that pool is released, so an abandoned request appears in the trace +instead of vanishing. + + + + + +flush the OpenTelemetry span pipeline before the router exits. Exit and +reload discarded the pipeline with up to 4096 spans still queued, losing every +span produced since the last export. The pipeline is now flushed with a 2 +second bound, so a slow or unreachable collector delays shutdown by at most +that long rather than either hanging it or costing the queued spans. + + + + + +rebuild the OpenTelemetry exporter only when the telemetry settings have +changed. Every configuration change rebuilt it, so adding a route or changing +an application stalled reconfiguration against an unreachable collector even +though it did not touch "/config/settings/telemetry". The exporter is now +rebuilt only when the endpoint, protocol, sampling ratio or batch size +actually differs from the running one. + + + + + +the OpenTelemetry span attributes "http.response.status_code" and +"http.request.body.size" are now emitted as OTLP integer values, as the HTTP +semantic conventions specify, instead of strings. A collector or query that +matches these attributes as strings must be adapted; one that reads them as +integers, or that relied on the convention rather than on Unit's previous +output, needs no change. + + + + + +the work a traced request does on the request path is reduced: attribute keys +are no longer copied onto the heap per request, the two attributes above no +longer round-trip through strings, and a span stage now crosses into the +tracer once instead of once per attribute. Together with the sampling fix +above, the telemetry overhead of a request measured 27% lower at +"sampling_ratio" 1.0 and 53% lower at 0.1. + + + + + +the Java module now bundles Apache Tomcat 9.0.121 (from 9.0.110) and +ClassGraph 4.8.186 (from 4.8.181). The upstream Tomcat 9.0.121 release fixes +CVE-2026-66299 (unbounded buffer for undelivered WebSocket messages, a memory +exhaustion by a slow client), CVE-2026-73180, CVE-2026-68569, CVE-2026-66422 +and CVE-2026-65927. FreeUnit bundles only the servlet, JSP and EL API jars, +Jasper and the Tomcat utility jars -- not catalina, the WebSocket +implementation or the example webapps -- so the bump keeps the jars it ships +on a supported release. + + + + + + + + + + +FreeUnit updated to 1.36.1. + + + From 4484b54021a24f8ab2693b5571a5890b26a771b7 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 20:40:27 +0200 Subject: [PATCH 052/391] 1.36.1 Bump FreeUnit to 1.36.1 (NXT_VERNUM 13601) in the version file and in the docs/unit-openapi.yaml title; the 1.36.1 changelog entries landed in the preceding commit. Co-Authored-By: Claude Opus --- docs/unit-openapi.yaml | 2 +- version | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index eb515305e..37938d7b0 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - title: "FreeUnit 1.36.0 (ex NGINX Unit)" + title: "FreeUnit 1.36.1 (ex NGINX Unit)" description: "FreeUnit (ex NGINX Unit) is a lightweight and versatile application runtime that provides the essential components for your web application as a single open-source server: running application code, serving static assets, diff --git a/version b/version index 5fb12c9d5..f702395d0 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ # Copyright (C) FreeUnit Community -NXT_VERSION=1.36.0 -NXT_VERNUM=13600 +NXT_VERSION=1.36.1 +NXT_VERNUM=13601 From bcf560a8cc6efd89958ce9d4fd0eafafef0bd284 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 20:41:23 +0200 Subject: [PATCH 053/391] chore(unitctl): bump crate versions to 1.36.1 Bump the unitctl crate versions to 1.36.1, refresh RELEASE-PROCESS.md's version-appendix example, and add the 1.36.1 entry and release links to the unitctl CHANGELOG (unitctl ships under its own unitctl/* tags, separate from the daemon release tag). Co-Authored-By: Claude Opus --- RELEASE-PROCESS.md | 6 +++--- tools/unitctl/CHANGELOG.md | 16 +++++++++++++++- tools/unitctl/Cargo.lock | 6 +++--- tools/unitctl/openapi-config.json | 2 +- tools/unitctl/unit-client-rs/Cargo.toml | 2 +- tools/unitctl/unit-openapi/Cargo.toml | 2 +- tools/unitctl/unit-openapi/README.md | 2 +- tools/unitctl/unitctl/Cargo.toml | 2 +- 8 files changed, 26 insertions(+), 12 deletions(-) diff --git a/RELEASE-PROCESS.md b/RELEASE-PROCESS.md index 77a07c67d..d4f2e5185 100644 --- a/RELEASE-PROCESS.md +++ b/RELEASE-PROCESS.md @@ -107,12 +107,12 @@ release number: # Copyright (C) FreeUnit Community - NXT_VERSION=1.36.0 - NXT_VERNUM=13600 + NXT_VERSION=1.36.1 + NXT_VERNUM=13601 - **`NXT_VERSION`** — human-readable dotted string (`MAJOR.MINOR.PATCH`). - **`NXT_VERNUM`** — the same number for compile-time comparisons: - `MAJOR * 10000 + MINOR * 100 + PATCH` (e.g. `1.36.0` → `13600`). Keep both in + `MAJOR * 10000 + MINOR * 100 + PATCH` (e.g. `1.36.1` → `13601`). Keep both in lockstep; a mismatch is a silent bug. ## Consumers diff --git a/tools/unitctl/CHANGELOG.md b/tools/unitctl/CHANGELOG.md index 7bf491ed6..67e1833d1 100644 --- a/tools/unitctl/CHANGELOG.md +++ b/tools/unitctl/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.36.1] - 2026-08-28 + +### Fixed +- Control API error responses that carry a `location` object (byte offset, line and column for + parse errors; the JSON Pointer `path` for validation errors) no longer fail to deserialise; + `unitctl` now prints the server's `detail`, `suggestion` and location instead of a generic JSON + decoding error. + +### Changed +- Version bump to track the FreeUnit 1.36.1 release. +- The `/status` counters in the OpenAPI spec are declared `format: int64`, so the generated + client models them as 64-bit instead of `i32`. + ## [1.36.0] - 2026-07-16 ### Added @@ -55,6 +68,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Previous releases See earlier versions for additional change history. -[Unreleased]: https://github.com/freeunitorg/freeunit/compare/unitctl/1.36.0...HEAD +[Unreleased]: https://github.com/freeunitorg/freeunit/compare/unitctl/1.36.1...HEAD +[1.36.1]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.36.1 [1.36.0]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.36.0 [1.35.5]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.35.5 diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 354c2d17d..5727c5a91 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -1831,7 +1831,7 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unit-client-rs" -version = "1.36.0" +version = "1.36.1" dependencies = [ "bollard", "bytes", @@ -1857,7 +1857,7 @@ dependencies = [ [[package]] name = "unit-openapi" -version = "1.36.0" +version = "1.36.1" dependencies = [ "base64", "bytes", @@ -1873,7 +1873,7 @@ dependencies = [ [[package]] name = "unitctl" -version = "1.36.0" +version = "1.36.1" dependencies = [ "bytes", "clap", diff --git a/tools/unitctl/openapi-config.json b/tools/unitctl/openapi-config.json index b8b428147..535313dd7 100644 --- a/tools/unitctl/openapi-config.json +++ b/tools/unitctl/openapi-config.json @@ -1,6 +1,6 @@ { "packageName": "unit-openapi", - "packageVersion": "1.36.0", + "packageVersion": "1.36.1", "library": "hyper", "preferUnsignedInt": true } diff --git a/tools/unitctl/unit-client-rs/Cargo.toml b/tools/unitctl/unit-client-rs/Cargo.toml index 28c879633..834c9b59d 100644 --- a/tools/unitctl/unit-client-rs/Cargo.toml +++ b/tools/unitctl/unit-client-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "unit-client-rs" -version = "1.36.0" +version = "1.36.1" authors = ["Elijah Zupancic"] edition = "2021" license = "Apache-2.0" diff --git a/tools/unitctl/unit-openapi/Cargo.toml b/tools/unitctl/unit-openapi/Cargo.toml index b529a5e24..4e4f14959 100644 --- a/tools/unitctl/unit-openapi/Cargo.toml +++ b/tools/unitctl/unit-openapi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "unit-openapi" -version = "1.36.0" +version = "1.36.1" authors = ["team@freeunit.org"] description = "FreeUnit (ex NGINX Unit) is a lightweight and versatile application runtime that provides the essential components for your web application as a single open-source server: running application code, serving static assets, handling TLS and request routing. **Important**: Unit's API is designed to expose any part of its configuration as an addressable endpoint. Suppose a JSON object is stored at `/config/listeners/`: ```json { \"*:8080\": { \"pass\": \"applications/wp_emea_dev\" } } ``` Here, `/config/listeners/_*:8080` and `/config/listeners/_*:8080/pass` are also endpoints. Generally, object options are addressable by their names, array items—by their indexes (`/array/0/`). **Note**: By default, Unit is configured through a UNIX domain socket. To use this specification with OpenAPI tools interactively, start Unit with a TCP port as the control socket." license = "Apache 2.0" diff --git a/tools/unitctl/unit-openapi/README.md b/tools/unitctl/unit-openapi/README.md index bb576e910..145a8b668 100644 --- a/tools/unitctl/unit-openapi/README.md +++ b/tools/unitctl/unit-openapi/README.md @@ -21,7 +21,7 @@ For more information, please visit [https://github.com/freeunitorg/freeunit](htt This API client was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the [openapi-spec](https://openapis.org) from a remote server, you can easily generate an API client. - API version: 0.2.0 -- Package version: 1.36.0 +- Package version: 1.36.1 - Generator version: 7.6.0 - Build package: `org.openapitools.codegen.languages.RustClientCodegen` diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index 676992e9c..67a64c6aa 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "unitctl" description = "CLI interface to the FreeUnit Control API" -version = "1.36.0" +version = "1.36.1" authors = ["Elijah Zupancic"] edition = "2021" license = "Apache-2.0" From 212ea0406bf7b3babd944473bef4c45e2a5580d5 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 20:41:38 +0200 Subject: [PATCH 054/391] chore(docker): regenerate Dockerfiles for 1.36.1 Regenerate pkg/docker/Dockerfile.* from template.Dockerfile so the committed image.version label and the `git clone -b ` source checkout track the 1.36.1 version bump. Co-Authored-By: Claude Opus --- pkg/docker/Dockerfile.go-1.25 | 4 ++-- pkg/docker/Dockerfile.go-1.26 | 4 ++-- pkg/docker/Dockerfile.java-17 | 4 ++-- pkg/docker/Dockerfile.java-21 | 4 ++-- pkg/docker/Dockerfile.java-25 | 4 ++-- pkg/docker/Dockerfile.java-26 | 4 ++-- pkg/docker/Dockerfile.minimal | 4 ++-- pkg/docker/Dockerfile.node-20 | 4 ++-- pkg/docker/Dockerfile.node-22 | 4 ++-- pkg/docker/Dockerfile.node-24 | 4 ++-- pkg/docker/Dockerfile.node-26 | 4 ++-- pkg/docker/Dockerfile.perl-5.38 | 4 ++-- pkg/docker/Dockerfile.perl-5.40 | 4 ++-- pkg/docker/Dockerfile.perl-5.42 | 4 ++-- pkg/docker/Dockerfile.php-8.3 | 4 ++-- pkg/docker/Dockerfile.php-8.4 | 4 ++-- pkg/docker/Dockerfile.php-8.5 | 4 ++-- pkg/docker/Dockerfile.python-3.12 | 4 ++-- pkg/docker/Dockerfile.python-3.12-slim | 4 ++-- pkg/docker/Dockerfile.python-3.13 | 4 ++-- pkg/docker/Dockerfile.python-3.13-slim | 4 ++-- pkg/docker/Dockerfile.python-3.14 | 4 ++-- pkg/docker/Dockerfile.python-3.14-slim | 4 ++-- pkg/docker/Dockerfile.ruby-3.3 | 4 ++-- pkg/docker/Dockerfile.ruby-3.4 | 4 ++-- pkg/docker/Dockerfile.ruby-4.0 | 4 ++-- pkg/docker/Dockerfile.wasm | 4 ++-- 27 files changed, 54 insertions(+), 54 deletions(-) diff --git a/pkg/docker/Dockerfile.go-1.25 b/pkg/docker/Dockerfile.go-1.25 index a917c3281..535c7f684 100644 --- a/pkg/docker/Dockerfile.go-1.25 +++ b/pkg/docker/Dockerfile.go-1.25 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.go-1.26 b/pkg/docker/Dockerfile.go-1.26 index d5330ff1e..a26d228eb 100644 --- a/pkg/docker/Dockerfile.go-1.26 +++ b/pkg/docker/Dockerfile.go-1.26 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-17 b/pkg/docker/Dockerfile.java-17 index 77142f771..12e935912 100644 --- a/pkg/docker/Dockerfile.java-17 +++ b/pkg/docker/Dockerfile.java-17 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-21 b/pkg/docker/Dockerfile.java-21 index 8a77c3eb4..9d1da9ea8 100644 --- a/pkg/docker/Dockerfile.java-21 +++ b/pkg/docker/Dockerfile.java-21 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-25 b/pkg/docker/Dockerfile.java-25 index 2e92da7ba..555a37af2 100644 --- a/pkg/docker/Dockerfile.java-25 +++ b/pkg/docker/Dockerfile.java-25 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-26 b/pkg/docker/Dockerfile.java-26 index 807e5178d..662519e60 100644 --- a/pkg/docker/Dockerfile.java-26 +++ b/pkg/docker/Dockerfile.java-26 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index 150793e58..2b2671742 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-20 b/pkg/docker/Dockerfile.node-20 index 215867433..e2b6d7272 100644 --- a/pkg/docker/Dockerfile.node-20 +++ b/pkg/docker/Dockerfile.node-20 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-22 b/pkg/docker/Dockerfile.node-22 index 8793cc9a1..69849a1d8 100644 --- a/pkg/docker/Dockerfile.node-22 +++ b/pkg/docker/Dockerfile.node-22 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-24 b/pkg/docker/Dockerfile.node-24 index 138ccdeb5..f0fc9737a 100644 --- a/pkg/docker/Dockerfile.node-24 +++ b/pkg/docker/Dockerfile.node-24 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-26 b/pkg/docker/Dockerfile.node-26 index da4d58ef7..fae55c414 100644 --- a/pkg/docker/Dockerfile.node-26 +++ b/pkg/docker/Dockerfile.node-26 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.38 b/pkg/docker/Dockerfile.perl-5.38 index fcce62dd0..79023c465 100644 --- a/pkg/docker/Dockerfile.perl-5.38 +++ b/pkg/docker/Dockerfile.perl-5.38 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.40 b/pkg/docker/Dockerfile.perl-5.40 index 90ae2fdef..fe2c3318d 100644 --- a/pkg/docker/Dockerfile.perl-5.40 +++ b/pkg/docker/Dockerfile.perl-5.40 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.42 b/pkg/docker/Dockerfile.perl-5.42 index 1efa58be8..04102f313 100644 --- a/pkg/docker/Dockerfile.perl-5.42 +++ b/pkg/docker/Dockerfile.perl-5.42 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.php-8.3 b/pkg/docker/Dockerfile.php-8.3 index 545834817..7d1962cce 100644 --- a/pkg/docker/Dockerfile.php-8.3 +++ b/pkg/docker/Dockerfile.php-8.3 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.php-8.4 b/pkg/docker/Dockerfile.php-8.4 index 94084852a..c8c63e239 100644 --- a/pkg/docker/Dockerfile.php-8.4 +++ b/pkg/docker/Dockerfile.php-8.4 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.php-8.5 b/pkg/docker/Dockerfile.php-8.5 index 23f7c69af..c8e812747 100644 --- a/pkg/docker/Dockerfile.php-8.5 +++ b/pkg/docker/Dockerfile.php-8.5 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.12 b/pkg/docker/Dockerfile.python-3.12 index a6dffb0a1..dc124fa63 100644 --- a/pkg/docker/Dockerfile.python-3.12 +++ b/pkg/docker/Dockerfile.python-3.12 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.12-slim b/pkg/docker/Dockerfile.python-3.12-slim index ca7821af3..b49a68774 100644 --- a/pkg/docker/Dockerfile.python-3.12-slim +++ b/pkg/docker/Dockerfile.python-3.12-slim @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.13 b/pkg/docker/Dockerfile.python-3.13 index aca936a09..a5d52b3f7 100644 --- a/pkg/docker/Dockerfile.python-3.13 +++ b/pkg/docker/Dockerfile.python-3.13 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.13-slim b/pkg/docker/Dockerfile.python-3.13-slim index 7127ab3fa..9f360f326 100644 --- a/pkg/docker/Dockerfile.python-3.13-slim +++ b/pkg/docker/Dockerfile.python-3.13-slim @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.14 b/pkg/docker/Dockerfile.python-3.14 index a684dfe82..5ac2e7b82 100644 --- a/pkg/docker/Dockerfile.python-3.14 +++ b/pkg/docker/Dockerfile.python-3.14 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.14-slim b/pkg/docker/Dockerfile.python-3.14-slim index 6241bda51..93fdf5a42 100644 --- a/pkg/docker/Dockerfile.python-3.14-slim +++ b/pkg/docker/Dockerfile.python-3.14-slim @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-3.3 b/pkg/docker/Dockerfile.ruby-3.3 index a5c2f876e..b8fe61c47 100644 --- a/pkg/docker/Dockerfile.ruby-3.3 +++ b/pkg/docker/Dockerfile.ruby-3.3 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-3.4 b/pkg/docker/Dockerfile.ruby-3.4 index 36ea02925..96139eba4 100644 --- a/pkg/docker/Dockerfile.ruby-3.4 +++ b/pkg/docker/Dockerfile.ruby-3.4 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-4.0 b/pkg/docker/Dockerfile.ruby-4.0 index 7d4ca2e38..8f64b4fab 100644 --- a/pkg/docker/Dockerfile.ruby-4.0 +++ b/pkg/docker/Dockerfile.ruby-4.0 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.wasm b/pkg/docker/Dockerfile.wasm index fa6dbec6b..83b35b516 100644 --- a/pkg/docker/Dockerfile.wasm +++ b/pkg/docker/Dockerfile.wasm @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.0" +LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -35,7 +35,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.0 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ From c1791326bf05768ca5507bd1fb6fe4a8b4343ac1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 00:03:18 +0200 Subject: [PATCH 055/391] docs(openapi): describe the /status telemetry object The 1.36.1 /status response carries a "telemetry" member reporting OpenTelemetry span export health, but the OpenAPI spec described only modules, connections, requests and applications, while documenting the /config/settings/telemetry side in full -- so the contract contradicted itself and generated clients had no way to know the member exists. Add the statusTelemetry and statusTelemetrySpans schemas and examples, list telemetry in the status schema and example in emission order, and document /status/telemetry, /status/telemetry/spans and its two leaves as endpoints, matching how the connections and requests subtrees are described. The controller resolves /status paths generically through nxt_conf_get_path(), so those leaves are addressable like the others. The four endpoints document a 404 as well as a 200, following the file's convention for a resource that need not exist: the member is absent unless Unit was built with OpenTelemetry, telemetry is configured and the exporter was built, and nxt_controller_status_response() answers "Invalid path." for a path that does not resolve. The schema records that absence, and that the counters are cumulative since the exporter was last built. Co-Authored-By: Claude Opus 5 --- docs/unit-openapi.yaml | 164 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 37938d7b0..335bcac2d 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -5176,6 +5176,120 @@ paths: Closed: value: 1307 + /status/telemetry: + summary: "Endpoint for the `telemetry` status object" + get: + operationId: getStatusTelemetry + summary: "Retrieve the telemetry status object" + description: "Retrieves the `telemetry` status object that represents + Unit's [OpenTelemetry export statistics] + (https://unit.nginx.org/statusapi/)." + + tags: + - status + + responses: + "200": + description: "OK; the `telemetry` object exists in the + configuration." + + content: + application/json: + schema: + $ref: "#/components/schemas/statusTelemetry" + + examples: + example1: + $ref: "#/components/examples/statusTelemetry" + + "404": + $ref: "#/components/responses/responseNotFound" + + /status/telemetry/spans: + summary: "Endpoint for the `spans` telemetry object" + get: + operationId: getStatusTelemetrySpans + summary: "Retrieve the telemetry spans object" + description: "Retrieves the `spans` object that represents Unit's + [OpenTelemetry span export counters] + (https://unit.nginx.org/statusapi/)." + + tags: + - status + + responses: + "200": + description: "OK; the `spans` object exists in the configuration." + + content: + application/json: + schema: + $ref: "#/components/schemas/statusTelemetrySpans" + + examples: + example1: + $ref: "#/components/examples/statusTelemetrySpans" + + "404": + $ref: "#/components/responses/responseNotFound" + + /status/telemetry/spans/exported: + summary: "Endpoint for the `exported` spans number" + get: + operationId: getStatusTelemetrySpansExported + summary: "Retrieve the exported spans number" + description: "Retrieves the `exported` spans number from Unit's + [telemetry statistics](https://unit.nginx.org/statusapi/), the spans + the collector accepted." + + tags: + - status + + responses: + "200": + description: "OK; the `exported` number exists in the + configuration." + + content: + application/json: + schema: + type: integer + + examples: + Exported: + value: 4096 + + "404": + $ref: "#/components/responses/responseNotFound" + + /status/telemetry/spans/failed: + summary: "Endpoint for the `failed` spans number" + get: + operationId: getStatusTelemetrySpansFailed + summary: "Retrieve the failed spans number" + description: "Retrieves the `failed` spans number from Unit's + [telemetry statistics](https://unit.nginx.org/statusapi/), the spans + whose export failed." + + tags: + - status + + responses: + "200": + description: "OK; the `failed` number exists in the configuration." + + content: + application/json: + schema: + type: integer + + examples: + Failed: + value: 0 + + "404": + $ref: "#/components/responses/responseNotFound" + /status/applications: summary: "Endpoint for the `applications` status object" get: @@ -6170,6 +6284,10 @@ components: closed: 1050 requests: total: 1307 + telemetry: + spans: + exported: 4096 + failed: 0 applications: wp: processes: @@ -6270,6 +6388,21 @@ components: value: total: 1307 + # /status/telemetry + statusTelemetry: + summary: "Regular telemetry status object" + value: + spans: + exported: 4096 + failed: 0 + + # /status/telemetry/spans + statusTelemetrySpans: + summary: "Regular telemetry spans status object" + value: + exported: 4096 + failed: 0 + # -- RESPONSES -- responses: @@ -7654,6 +7787,9 @@ components: requests: $ref: "#/components/schemas/statusRequests" + telemetry: + $ref: "#/components/schemas/statusTelemetry" + applications: $ref: "#/components/schemas/statusApplications" @@ -7733,6 +7869,34 @@ components: type: integer description: "Total non-API requests during the instance’s lifetime." + # /status/telemetry + statusTelemetry: + description: "Represents the health of Unit's OpenTelemetry span + export. The object is omitted entirely unless Unit was built with + OpenTelemetry support, `/config/settings/telemetry` is set, and the + exporter was built successfully, so its absence is how a client tells + telemetry that is off from telemetry that is idle." + type: object + properties: + spans: + $ref: "#/components/schemas/statusTelemetrySpans" + + # /status/telemetry/spans + statusTelemetrySpans: + description: "Represents Unit's OpenTelemetry span export counters. + Both are cumulative since the exporter was last built, so changing + any telemetry setting restarts them, while re-applying an identical + configuration does not." + type: object + properties: + exported: + type: integer + description: "Spans the collector accepted." + + failed: + type: integer + description: "Spans whose export failed." + # /status/connections statusConnections: description: "Represents Unit's per-instance connection statistics." From 8d82e4a944f3181913744657790038281507f0d9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 00:36:49 +0200 Subject: [PATCH 056/391] docs(openapi): model the control API error response Every error response in the spec points at jsonErrorMessage, which declared the body to be an object of strings and nothing more. The server has never sent only that: "location" is a nested object carrying integer offset, line and column, and 1.36.1 added an RFC 6901 JSON Pointer in "location"."path" plus a top-level "suggestion" naming the parameter an unknown one was probably a typo of. The spec contradicted itself as a result -- its own errorInvalidJson example shows a location object, which the schema it is attached to forbids. Give jsonErrorMessage the members the controller actually sets, keeping additionalProperties so anything not modelled here stays legal, and add errorLocation for the nested object, documenting for each member the condition under which it appears: offset, line and column only for an error found while parsing, path wherever the failure can be placed. Add an errorUnknownParameter example carrying the two new members and offer it on 400 beside the existing parse-error example, so the pointer and the suggestion appear in the rendered documentation. jsonSuccessMessage is deliberately left alone. A named schema that is a pure additionalProperties map generates as HashMap, and unit_client.rs hard-codes that type for the restart endpoint; giving it properties could turn it into a struct and break the build for no gain. Error bodies never reach a generated model -- the client templates turn a non-2xx into an error carrying raw bytes -- so this change cannot affect it. Validated by checking every committed example against the schema it is attached to: 60 example sites failed before this change and 6 after, with no site newly failing. The 6 that remain are unrelated /config application and settings examples. Co-Authored-By: Claude Opus 5 --- docs/unit-openapi.yaml | 61 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 335bcac2d..626e6131d 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -5609,6 +5609,15 @@ components: line: 1 column: 0 + errorUnknownParameter: + summary: "400 error response for an unknown parameter" + value: + error: "Invalid configuration." + detail: 'Unknown parameter "pas".' + location: + path: "/routes/0/action" + suggestion: "pass" + errorValueDoesntExist: summary: "404 error response" value: @@ -6442,6 +6451,9 @@ components: example1: $ref: "#/components/examples/errorInvalidJson" + example2: + $ref: "#/components/examples/errorUnknownParameter" + responseNotFound: description: "Not Found; the value does not exist in the configuration. This may occur if any part of the path is non-existent." @@ -6493,10 +6505,57 @@ components: jsonErrorMessage: type: object - description: "JSON message on error." + description: "JSON message on error. Only `error` is always present; + each of the others is omitted where it does not apply." + properties: + error: + type: string + description: "What went wrong, in a human-readable sentence." + + detail: + type: string + description: "A longer explanation of the error, where one adds + anything to the message." + + location: + $ref: "#/components/schemas/errorLocation" + + suggestion: + type: string + description: "The name the request most likely meant, reported when + an unknown parameter is a close and unambiguous typo of a known + one." + additionalProperties: type: string + errorLocation: + type: object + description: "Where in the request the error is. Present when the error + can be placed; which members appear depends on how it was found." + properties: + offset: + type: integer + description: "Byte offset into the request body, reported only for + an error found while parsing it." + + line: + type: integer + description: "Line number, reported with `column` and only for an + error found while parsing." + + column: + type: integer + description: "Column number, reported with `line`." + + path: + type: string + description: "An RFC 6901 JSON Pointer to where the error is: the + member whose value failed validation, the member a required + parameter is missing from, or the containing object when the + failure is an unknown parameter, whose name the message itself + carries. An empty string is the document root." + # Configuration sections as data types; hugely reliant on each other # /certificates From ee850abd12aaf686d1ab751581c787c510049f95 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 01:02:19 +0200 Subject: [PATCH 057/391] fix(unitctl): stop hiding the server's error message A control API error body is not a map of strings: "location" is a nested object. unit_client.rs deserialized every non-2xx body into HashMap, so any response carrying it failed serde and unitctl reported a JSON deserialization error instead of what the server said. Until 1.36.1 that only hit JSON parse errors, which are rare and already obvious. 1.36.1 puts an RFC 6901 pointer in "location"."path" on every validation error, so the ordinary case -- a typo in a configuration -- now loses the server's explanation, including the "suggestion" naming the parameter the typo probably meant, which exists precisely to help with it. Deserialize into serde_json::Value and pull the parts out in describe_error_body(), which handles every shape the controller can send: a pointer at the member at fault, the empty pointer that means the document root, and the line and column an error found while parsing places itself by instead of a pointer. The two new members are appended to the existing message, so the output is unchanged for a response that carries neither: HTTP response error [path=/config, status=400 Bad Request]: Error: Invalid configuration. Detail: Unknown parameter "pas". Location: /routes/0/action Did you mean: pass The extraction is a free function so it can be tested without a live control socket, which the crate's existing tests all need; the table test covers the shapes above plus a body that is not an object. unit-openapi/README.md is regenerated: it lists the endpoints and models the spec gained in this branch. Verified by generating the client with the pinned 7.6.0 generator and running cargo check and clippy over the workspace, then cargo test: 26 passed and 3 failed, the same 3 that fail on the unmodified base because they need a live control socket. Co-Authored-By: Claude Opus 5 --- .../unitctl/unit-client-rs/src/unit_client.rs | 121 +++++++++++++++++- tools/unitctl/unit-openapi/README.md | 8 ++ 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/tools/unitctl/unit-client-rs/src/unit_client.rs b/tools/unitctl/unit-client-rs/src/unit_client.rs index 55ff72ad8..e469546d1 100644 --- a/tools/unitctl/unit-client-rs/src/unit_client.rs +++ b/tools/unitctl/unit-client-rs/src/unit_client.rs @@ -43,7 +43,9 @@ custom_error! {pub UnitClientError HttpResponseJsonBodyError { status: http::StatusCode, path: String, error: String, - detail: String} = "HTTP response error [path={path}, status={status}]:\n Error: {error}\n Detail: {detail}", + detail: String, + location: String, + suggestion: String} = "HTTP response error [path={path}, status={status}]:\n Error: {error}\n Detail: {detail}{location}{suggestion}", IoError { source: io::Error, socket: String } = "IO error [socket={socket}]", UnixSocketAddressError { source: io::Error, @@ -165,6 +167,50 @@ pub struct UnitClient { client: Box, } +/// Pulls the parts of a control API error body out for display: the message, the +/// detail, where the error is, and the name a typo probably meant. The last two +/// come back pre-formatted, empty when they do not apply, so a response carrying +/// neither renders exactly as it did before they existed. +fn describe_error_body(body: &serde_json::Value) -> (String, String, String, String) { + let member = |name: &str| { + body.get(name) + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string() + }; + + let mut error = member("error"); + if error.is_empty() { + error = "Unknown error".to_string(); + } + + // A validation error places itself with an RFC 6901 pointer, where the empty + // string is the document root; an error found while parsing the request has + // no pointer and places itself by position instead. + let place = body.get("location").and_then(|location| { + let number = |name: &str| location.get(name).and_then(serde_json::Value::as_i64); + + match location.get("path").and_then(serde_json::Value::as_str) { + Some("") => Some("the document root".to_string()), + Some(pointer) => Some(pointer.to_string()), + None => match (number("line"), number("column"), number("offset")) { + (Some(line), Some(column), _) => Some(format!("line {}, column {}", line, column)), + (_, _, Some(offset)) => Some(format!("byte offset {}", offset)), + _ => None, + }, + } + }); + + let location = place.map(|place| format!("\n Location: {}", place)).unwrap_or_default(); + + let suggestion = match member("suggestion").as_str() { + "" => String::new(), + name => format!("\n Did you mean: {}", name), + }; + + (error, member("detail"), location, suggestion) +} + impl UnitClient { pub fn new(control_socket: ControlSocket) -> Self { if control_socket.is_local_socket() { @@ -228,17 +274,27 @@ impl UnitClient { let mut reader = body_bytes.reader(); if !status.is_success() { - let error: HashMap = + // The control API answers an error with "error" and, where they + // apply, "detail", a "location" object placing the error, and a + // "suggestion" naming the parameter an unknown one was probably a + // typo of. "location" is an object, so the body is not a map of + // strings and must not be deserialized as one: that fails the + // whole body and hides the message it carries. + let body: serde_json::Value = serde_json::from_reader(&mut reader).map_err(|error| UnitClientError::JsonError { source: error, path: path.to_string(), })?; + let (error, detail, location, suggestion) = describe_error_body(&body); + return Err(UnitClientError::HttpResponseJsonBodyError { status, path: path.to_string(), - error: error.get("error").unwrap_or(&"Unknown error".into()).to_string(), - detail: error.get("detail").unwrap_or(&"".into()).to_string(), + error, + detail, + location, + suggestion, }); } serde_json::from_reader(&mut reader).map_err(|error| UnitClientError::JsonError { @@ -370,6 +426,63 @@ mod tests { use crate::unitd_instance::UnitdInstance; use super::*; + + /// The shapes nxt_controller_response() can send, and how each renders. + #[test] + fn describes_every_error_body_shape() { + let case = |body: serde_json::Value| { + let (error, detail, location, suggestion) = describe_error_body(&body); + format!("{}|{}|{}|{}", error, detail, location, suggestion) + }; + + // A plain error, as before location and suggestion existed: both parts + // must be empty so the rendered message is unchanged. + assert_eq!( + case(serde_json::json!({"error": "Value doesn't exist."})), + "Value doesn't exist.|||" + ); + + // A validation error: an RFC 6901 pointer at the member at fault. + assert_eq!( + case(serde_json::json!({ + "error": "Invalid configuration.", + "detail": "Unknown parameter \"pas\".", + "location": {"path": "/routes/0/action"}, + "suggestion": "pass" + })), + "Invalid configuration.|Unknown parameter \"pas\".|\n Location: /routes/0/action|\n Did you mean: pass" + ); + + // The empty pointer is the document root, not a missing value. + assert_eq!( + case(serde_json::json!({"error": "Invalid configuration.", + "location": {"path": ""}})), + "Invalid configuration.||\n Location: the document root|" + ); + + // An error found while parsing places itself by position: no pointer. + assert_eq!( + case(serde_json::json!({"error": "Invalid JSON.", + "location": {"offset": 12, "line": 2, "column": 5}})), + "Invalid JSON.||\n Location: line 2, column 5|" + ); + + // Offset without line/column, which the controller sends when the + // position is known but not the line. + assert_eq!( + case(serde_json::json!({"error": "Invalid JSON.", "location": {"offset": 12}})), + "Invalid JSON.||\n Location: byte offset 12|" + ); + + // Nothing usable in location, and a body that is not an object at all: + // neither may panic or invent a location. + assert_eq!( + case(serde_json::json!({"error": "Invalid JSON.", "location": {}})), + "Invalid JSON.|||" + ); + assert_eq!(case(serde_json::json!("not an object")), "Unknown error|||"); + } + // Integration tests #[tokio::test] diff --git a/tools/unitctl/unit-openapi/README.md b/tools/unitctl/unit-openapi/README.md index 145a8b668..dfddcbd67 100644 --- a/tools/unitctl/unit-openapi/README.md +++ b/tools/unitctl/unit-openapi/README.md @@ -364,6 +364,10 @@ Class | Method | HTTP request | Description *StatusApi* | [**get_status_modules_lang_version**](docs/StatusApi.md#get_status_modules_lang_version) | **Get** /status/modules/{langMod}/version | Retrieve the language module version object *StatusApi* | [**get_status_requests**](docs/StatusApi.md#get_status_requests) | **Get** /status/requests | Retrieve the requests status object *StatusApi* | [**get_status_requests_total**](docs/StatusApi.md#get_status_requests_total) | **Get** /status/requests/total | Retrieve the total requests number +*StatusApi* | [**get_status_telemetry**](docs/StatusApi.md#get_status_telemetry) | **Get** /status/telemetry | Retrieve the telemetry status object +*StatusApi* | [**get_status_telemetry_spans**](docs/StatusApi.md#get_status_telemetry_spans) | **Get** /status/telemetry/spans | Retrieve the telemetry spans object +*StatusApi* | [**get_status_telemetry_spans_exported**](docs/StatusApi.md#get_status_telemetry_spans_exported) | **Get** /status/telemetry/spans/exported | Retrieve the exported spans number +*StatusApi* | [**get_status_telemetry_spans_failed**](docs/StatusApi.md#get_status_telemetry_spans_failed) | **Get** /status/telemetry/spans/failed | Retrieve the failed spans number *TlsApi* | [**delete_listener_tls**](docs/TlsApi.md#delete_listener_tls) | **Delete** /config/listeners/{listenerName}/tls | Delete the tls object in a listener *TlsApi* | [**delete_listener_tls_certificate**](docs/TlsApi.md#delete_listener_tls_certificate) | **Delete** /config/listeners/{listenerName}/tls/certificate/{arrayIndex} | Delete a certificate array item in a listener *TlsApi* | [**delete_listener_tls_certificates**](docs/TlsApi.md#delete_listener_tls_certificates) | **Delete** /config/listeners/{listenerName}/tls/certificate | Delete the certificate option in a listener @@ -465,6 +469,8 @@ Class | Method | HTTP request | Description - [ConfigSettingsHttpStatic](docs/ConfigSettingsHttpStatic.md) - [ConfigSettingsHttpStaticMimeType](docs/ConfigSettingsHttpStaticMimeType.md) - [ConfigSettingsTelemetry](docs/ConfigSettingsTelemetry.md) + - [ErrorLocation](docs/ErrorLocation.md) + - [JsonErrorMessage](docs/JsonErrorMessage.md) - [Status](docs/Status.md) - [StatusApplicationsApp](docs/StatusApplicationsApp.md) - [StatusApplicationsAppProcesses](docs/StatusApplicationsAppProcesses.md) @@ -472,6 +478,8 @@ Class | Method | HTTP request | Description - [StatusConnections](docs/StatusConnections.md) - [StatusModulesLang](docs/StatusModulesLang.md) - [StatusRequests](docs/StatusRequests.md) + - [StatusTelemetry](docs/StatusTelemetry.md) + - [StatusTelemetrySpans](docs/StatusTelemetrySpans.md) - [StringOrStringArray](docs/StringOrStringArray.md) From 78caadd7b386a268d6a531bfd919bc86735aa469 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 01:59:06 +0200 Subject: [PATCH 058/391] docs(openapi): mark error required, and say which document each location indexes Three points from review, each checked against the controller. "error" is the one member every error response carries: nxt_controller_ response() writes it for any status >= 400 before anything optional, and the schema's description already said so. Without a required list a generated client types it Option, so every consumer handles an absence that cannot happen. "path" and the numeric members do not index the same document, which the old wording hid by calling the empty pointer "the document root". A request to a path below "/config" is validated against a clone of the whole configuration with the change applied (nxt_controller_process_ config()), so the pointer is rooted at the configuration, not at the request body -- a client that resolves it against what it sent looks in the wrong document. The numeric members do index the request body. The parse offset is measured after a leading UTF-8 byte order mark is skipped, so an offset does not count the bytes a client sent when it sent a BOM. Appended rather than folded: this branch is already merged into by #241 and rewriting it would strand that history. Every committed example still validates: 6 failing sites, the same unrelated /config ones, none new. Co-Authored-By: Claude Opus 5 --- docs/unit-openapi.yaml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 626e6131d..bfaf16c4f 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -6507,6 +6507,8 @@ components: type: object description: "JSON message on error. Only `error` is always present; each of the others is omitted where it does not apply." + required: + - error properties: error: type: string @@ -6531,13 +6533,17 @@ components: errorLocation: type: object - description: "Where in the request the error is. Present when the error - can be placed; which members appear depends on how it was found." + description: "Where the error is. Present when it can be placed. Which + members appear depends on how it was found, and they do not index the + same document: `offset`, `line` and `column` locate the error in the + request body, while `path` locates it in the resulting configuration." properties: offset: type: integer description: "Byte offset into the request body, reported only for - an error found while parsing it." + an error found while parsing it. A leading UTF-8 byte order mark + is skipped before parsing, so the offset is relative to the JSON + that follows it rather than to the bytes sent." line: type: integer @@ -6550,11 +6556,15 @@ components: path: type: string - description: "An RFC 6901 JSON Pointer to where the error is: the - member whose value failed validation, the member a required - parameter is missing from, or the containing object when the - failure is an unknown parameter, whose name the message itself - carries. An empty string is the document root." + description: "An RFC 6901 JSON Pointer to the member whose value + failed validation, to the member a required parameter is missing + from, or to the containing object when the failure is an unknown + parameter, whose name the message itself carries. It points into + the configuration the request would produce, not into the request + body: a request to a path below \"/config\" is validated against a + copy of the whole configuration with the change applied, so the + pointer is rooted at the configuration and an empty string is the + configuration itself." # Configuration sections as data types; hugely reliant on each other From d5a6a4615b2fdd6b266947deee77bb3ca8cec08d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 22:55:11 +0000 Subject: [PATCH 059/391] build(deps): bump actions/setup-java from 5 to 6 Bumps [actions/setup-java](https://github.com/actions/setup-java) from 5 to 6. - [Release notes](https://github.com/actions/setup-java/releases) - [Commits](https://github.com/actions/setup-java/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-java dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-test.yml | 2 +- .github/workflows/unitctl.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index facc2c403..0e3cd85de 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -263,7 +263,7 @@ jobs: ## Java ## - - uses: actions/setup-java@v5 + - uses: actions/setup-java@v6 with: distribution: temurin java-version: '${{ steps.metadata.outputs.version }}' diff --git a/.github/workflows/unitctl.yml b/.github/workflows/unitctl.yml index 8d285ec36..a0f35b57f 100644 --- a/.github/workflows/unitctl.yml +++ b/.github/workflows/unitctl.yml @@ -60,7 +60,7 @@ jobs: brew install make gnu-sed grep gawk echo "MAKE=gmake" >> $GITHUB_ENV - - uses: actions/setup-java@v5 + - uses: actions/setup-java@v6 with: distribution: temurin java-version: 21 @@ -137,7 +137,7 @@ jobs: brew install make gnu-sed grep gawk echo "MAKE=gmake" >> $GITHUB_ENV - - uses: actions/setup-java@v5 + - uses: actions/setup-java@v6 with: distribution: temurin java-version: 21 From 10f0892f3bd1424d72c69278aa961734e88ed5ef Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 03:57:23 +0200 Subject: [PATCH 060/391] docs: state the OpenSSL 1.1.1 floor rationale without expired claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The floor's justification named three platforms as shipping a vendor-patched 1.1.1: RHEL 8, Amazon Linux 2 and Debian 11. Two of them are ending. Amazon Linux 2 reached EOL on 2026-06-30 and Debian 11's LTS ends 2026-08-31, so from September RHEL 8 is the only platform in the matrix still receiving vendor patches for it, and Red Hat maintains it into 2029. Say that, with the dates rather than a tense: for the last days of August, Debian 11 is still receiving them, and a sentence claiming otherwise would be wrong on the day it merged. Amazon Linux 2 and Debian 11 stay supported either way, but on FreeUnit's own three-year post-EOL grace window rather than on any vendor's patches -- a different claim, now made separately. Correct the package attribution while here: pkg/rpm selects openssl11-devel only for Amazon Linux 2 (pkg/rpm/Makefile:33-37, pkg/rpm/unit.spec.in:7-11); RHEL 8 builds against openssl-devel, and Debian is not rpm at all. The tables the rationale cites did not mark Amazon Linux 2 as past upstream EOL while the text leaned on exactly that status, so add the "(EOL) †" marker in EOL.md and "note": "EOL" in pkg/eol.json, matching Debian 11, Fedora 40-42 and Alpine 3.20. That note changes no checker output -- the "add (EOL) flag" warning it suppresses is raised only for runtimes (pkg/eol/src/main.rs:594-612), while an OS past its date takes the "plan migration" path, which no flag silences. Verified by running the checker both ways: 14 warnings before and after, the amazonlinux line identical. The marker is for consistency with every other past-EOL row. Dates verified against endoflife.date: amazon-linux 2 eol 2026-06-30; debian 11 eol 2024-08-14 with LTS to 2026-08-31; rhel 8 maintenance to 2029-05-31 (ELS 2033-05-31). Co-Authored-By: Claude Opus 5 --- EOL.md | 4 ++-- auto/ssltls | 12 ++++++++---- pkg/eol.json | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/EOL.md b/EOL.md index 098fcbe01..997fe8a26 100644 --- a/EOL.md +++ b/EOL.md @@ -46,7 +46,7 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Fedora | 44 | 2027-06 | 2030-06 | 3.14 | | CentOS Stream | 9 | 2027-05 | 2030-05 | 3.11 | | CentOS Stream | 10 | 2030-05 | 2033-05 | 3.13 | -| Amazon Linux | 2 | 2026-06 | 2029-06 | 3.7 ‡ | +| Amazon Linux | 2 (EOL) † | 2026-06 | 2029-06 | 3.7 ‡ | | Amazon Linux | 2023 | 2029-06 | 2032-06 | 3.11 | | Ubuntu (LTS) | 22.04 | 2027-04 | 2030-04 | 3.10 | | Ubuntu (LTS) | 24.04 | 2029-05 | 2032-05 | 3.12 | @@ -78,7 +78,7 @@ needs an upgrade or replacement decision instead. | Dependency | Role | Floor / Version | Upstream EOL | Notes | |------------|------|-----------------|--------------|-------| -| OpenSSL | TLS backend (`--openssl`) | 1.1.1 (floor) | Sep 2023 | Kept because RHEL 8, Amazon Linux 2 (`openssl11-devel`) and Debian 11 ship it. Floor declared by the `auto/ssltls` probe (PR #224). | +| OpenSSL | TLS backend (`--openssl`) | 1.1.1 (floor) | Sep 2023 | The floor stays at 1.1.1 rather than 3.x because RHEL 8 still ships a vendor-patched 1.1.1 and Red Hat maintains it into 2029 -- from 2026-09 it is the last platform in the matrix still receiving vendor patches for it. Amazon Linux 2's ended 2026-06-30 and Debian 11's LTS ends 2026-08-31; both remain inside FreeUnit's own three-year post-EOL support window (see `_grace_os` in `pkg/eol.json`). Floor declared by the `auto/ssltls` probe (PR #224). | | OpenSSL | tested ceiling | 3.6.2 (CI build) | Nov 2026 | `build-test.yml` builds it into `/opt/openssl-3.6` (`OPENSSL_VERSION`). **The pin needs a bump before 2026-11-01** (3.5 is the LTS, EOL Apr 2030). | | OpenSSL | tested ceiling | 4.0.2 (CI build) | May 2027 | The `openssl4` job builds it into `/opt/openssl-4.0` (`OPENSSL4_VERSION`) and compiles with `-DOPENSSL_NO_DEPRECATED`, so any use of an API 4.0 deprecates fails the build. | | Apache Tomcat | Servlet/JSP/EL API + Jasper jars bundled by the Java module | 9.0.x | Mar 2027 ([no earlier than](https://tomcat.apache.org/whichversion.html)) | Bundled by `auto/modules/java`; independent of the JDK variant. | diff --git a/auto/ssltls b/auto/ssltls index f54aab63e..c4447e0c2 100644 --- a/auto/ssltls +++ b/auto/ssltls @@ -41,10 +41,14 @@ if [ $NXT_OPENSSL = YES ]; then # as well; that is deliberate, so that the floor probe below is the # one to report a too-old library by name instead of configure # bailing out with a misleading "no OpenSSL library found". The - # floor is not a build fact but a support decision: 1.1.1 is - # vendor-maintained on RHEL 8, Amazon Linux 2 and Debian 11, all - # still in the OS matrix (pkg/eol.json), and pkg/rpm builds against - # openssl11-devel there. + # floor is not a build fact but a support decision: it stays + # at 1.1.1 rather than 3.x because RHEL 8 still ships a + # vendor-patched 1.1.1 that Red Hat maintains into 2029, and from + # 2026-09 it is the last platform in the matrix still receiving + # those patches: Amazon Linux 2's ended 2026-06-30 and Debian + # 11's LTS ends 2026-08-31. Both stay in FreeUnit's own grace + # window; the OS matrix and that window both live in + # pkg/eol.json (see EOL.md). # Checking it here fails configure with a clear message instead of # leaving a too-old library to surface as an implicit declaration # deep in the build. diff --git a/pkg/eol.json b/pkg/eol.json index 53bca3b01..7302e2ffd 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -18,7 +18,7 @@ {"version": "10", "eol": "2030-05", "supported_until": "2033-05", "default_python": "3.13"} ], "amazonlinux": [ - {"version": "2", "eol": "2026-06", "supported_until": "2029-06", "default_python": "3.7", "default_python_note": "OS-shipped; Python 3.7 upstream EOL Jun 2023"}, + {"version": "2", "eol": "2026-06", "supported_until": "2029-06", "default_python": "3.7", "default_python_note": "OS-shipped; Python 3.7 upstream EOL Jun 2023", "note": "EOL"}, {"version": "2023", "eol": "2029-06", "supported_until": "2032-06", "default_python": "3.11"} ], "ubuntu": [ @@ -94,7 +94,7 @@ "role": "TLS backend (./configure --openssl)", "floor": "1.1.1", "floor_eol": "2023-09", - "floor_note": "Upstream-EOL but still supported by FreeUnit because it is what RHEL 8, Amazon Linux 2 (openssl11-devel, see pkg/rpm/Makefile and pkg/rpm/unit.spec.in) and Debian 11 ship, and all three are inside the OS support window above.", + "floor_note": "The floor stays at 1.1.1 rather than 3.x because RHEL 8 (openssl-devel) still ships a vendor-patched 1.1.1 that Red Hat maintains into 2029, and from 2026-09 it is the last platform in the matrix still receiving those patches; Amazon Linux 2 builds against openssl11-devel (see pkg/rpm/Makefile and pkg/rpm/unit.spec.in). Amazon Linux 2 and Debian 11 remain inside FreeUnit's own three-year post-EOL support window (see _grace_os).", "floor_declared_by": "auto/ssltls probe (PR #224), which also drops the pre-1.1.0 compatibility code in src/nxt_openssl.c", "tested_ceiling": "3.6.2 and 4.0.2", "tested_ceiling_note": "build-test.yml tests two ceilings from source: 3.6.2 (OPENSSL_VERSION env, built into /opt/openssl-3.6) and 4.0.2 (OPENSSL4_VERSION env, /opt/openssl-4.0, built by the openssl4 job with --cc-opt=-DOPENSSL_NO_DEPRECATED so any use of an API OpenSSL 4.0 deprecates fails the build). ACTION: OpenSSL 3.6 goes upstream-EOL 2026-11-01, so the OPENSSL_VERSION pin needs a bump to a supported branch (3.5 is the LTS, EOL 2030-04); 4.0 is supported to 2027-05.", From 90c5ffb8df6b4594826cdaa48f293839ca83ab52 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 05:16:10 +0200 Subject: [PATCH 061/391] docs(openapi): declare the /status counters as 64-bit src/nxt_status.h stores accepted_conns, idle_conns, closed_conns, requests, otel_spans_exported, and otel_spans_failed as uint64_t, but docs/unit-openapi.yaml declared them as bare `integer`. openapi-generator's Rust template maps that to Option, so a counter above 2,147,483,647 fails to deserialise in the generated client (found by GitHub Codex on PR #237, review 5056461539). Add `format: int64` (which the generator maps to i64) to: - statusConnections.accepted / .idle / .closed - statusRequests.total - statusTelemetrySpans.exported / .failed on both the named component schemas and the inline response schemas of their leaf endpoints (/status/connections/accepted|idle|closed, /status/requests/total, /status/telemetry/spans/exported|failed). `active` connections is a computed value (accepted - closed - idle), not a stored struct field, and is left untouched here. active_requests, pending_processes, processes, and idle_processes stay bare `integer` since they are uint32_t in nxt_status_app_t. No `minimum: 0` is added: tools/unitctl/openapi-config.json sets preferUnsignedInt, and pairing that with a minimum would flip these to u64 in the generated client instead of the intended i64. Verification needs regenerating the client with openapi-generator-cli 7.6.0 (Java, not available in this environment) and checking tools/unitctl/unit-openapi/src/models/status_connections.rs and status_telemetry_spans.rs for i64; that lands via the next `make openapi-generate`. --- docs/unit-openapi.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index bfaf16c4f..3c0d5a32c 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -5051,6 +5051,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Accepted: @@ -5075,6 +5076,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Active: @@ -5098,6 +5100,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Idle: @@ -5122,6 +5125,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Closed: @@ -5171,6 +5175,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Closed: @@ -5254,6 +5259,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Exported: @@ -5282,6 +5288,7 @@ paths: application/json: schema: type: integer + format: int64 examples: Failed: @@ -7936,6 +7943,7 @@ components: properties: total: type: integer + format: int64 description: "Total non-API requests during the instance’s lifetime." # /status/telemetry @@ -7960,10 +7968,12 @@ components: properties: exported: type: integer + format: int64 description: "Spans the collector accepted." failed: type: integer + format: int64 description: "Spans whose export failed." # /status/connections @@ -7973,19 +7983,23 @@ components: properties: accepted: type: integer + format: int64 description: "Total accepted connections during the instance’s lifetime." active: type: integer + format: int64 description: "Current active connections for the instance." idle: type: integer + format: int64 description: "Current idle connections for the instance." closed: type: integer + format: int64 description: "Total closed connections during the instance’s lifetime." From 148cc0b10f423c8e871ccf5a52b682015532d0f3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 05:12:30 +0200 Subject: [PATCH 062/391] ci: cancel superseded PR runs, and bound every job No workflow declared a concurrency group, so a push to a pull request left its predecessor running to completion. Measured on this repo: PR #243 ran Build & Test five times between 01:39 and 02:23 and cancelled none of them -- four superseded 26-job matrices, about 450 job-minutes, inside 45 minutes. The cost is not billing. This repository is public and runs on standard runners, so the minutes are free (the timing API reports billable UBUNTU totals of zero). The cost is the runner pool: those superseded matrices are what turned a 14-17 minute Build & Test into 44-57 minutes during last night's release, with one run queued for an hour and 48 minutes before it started. Everyone's pull requests slowed down, including the ones nobody had superseded. Cancel only pull request runs. Non-PR events fall back to the run id rather than the ref, so each push, tag, schedule and dispatch gets a group of its own. Grouping them by ref would have been actively harmful: Actions keeps only one PENDING run per group and replaces it when another arrives, even with cancel-in-progress false, so a burst of master pushes would have silently dropped intermediate commits' runs, and a queued unitctl release dispatch could have been displaced by an unrelated push. The tag-driven workflows -- build-deb, release-docker, build-distros -- get no group at all. Bound every job while here. Nothing in the repository set timeout-minutes, so a hung job held a runner for the 360-minute default and fed the same saturation. For the 22 jobs with run history the value is at least twice the slowest observed execution -- 30 for the build-test matrix against a 12.7-minute worst case, 40 for sanitize, 90 for the docker image builds. timeout-minutes bounds execution and not queue time, so a loaded-but-healthy run cannot trip one. The two exceptions are guesses with nothing to measure: claude.yml has only skipped runs and fuzzing.yml has never run at all. Co-Authored-By: Claude Opus 5 --- .github/workflows/analysis-clang-ast.yml | 9 +++++++++ .github/workflows/build-deb.yml | 4 ++++ .github/workflows/build-distros.yml | 2 ++ .github/workflows/build-test.yml | 12 ++++++++++++ .github/workflows/claude.yml | 2 ++ .github/workflows/eol-check.yml | 9 +++++++++ .github/workflows/fuzzing.yml | 1 + .github/workflows/lint-whitespace.yml | 9 +++++++++ .github/workflows/release-docker.yml | 3 +++ .github/workflows/sanitize.yml | 9 +++++++++ .github/workflows/security-seccomp.yml | 9 +++++++++ .github/workflows/unitctl.yml | 11 +++++++++++ 12 files changed, 80 insertions(+) diff --git a/.github/workflows/analysis-clang-ast.yml b/.github/workflows/analysis-clang-ast.yml index cf4d61305..5ce5c6a19 100644 --- a/.github/workflows/analysis-clang-ast.yml +++ b/.github/workflows/analysis-clang-ast.yml @@ -20,9 +20,18 @@ on: - 'pkg/contrib/**' - '.github/workflows/analysis-clang-ast.yml' +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: clang-ast: runs-on: ubuntu-latest + timeout-minutes: 20 container: image: debian:testing diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 3539fe0e5..32a940176 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -69,6 +69,7 @@ env: jobs: build-trixie: runs-on: ubuntu-latest + timeout-minutes: 60 container: image: debian:trixie @@ -160,6 +161,7 @@ jobs: smoke-test: needs: build-trixie runs-on: ubuntu-latest + timeout-minutes: 20 container: image: debian:trixie strategy: @@ -298,6 +300,7 @@ jobs: package-qa: needs: build-trixie runs-on: ubuntu-latest + timeout-minutes: 20 container: image: debian:trixie strategy: @@ -355,6 +358,7 @@ jobs: needs: [build-trixie, smoke-test, package-qa] if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: write # required to upload assets to the release steps: diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 1255c319b..7055ee19d 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -23,6 +23,7 @@ jobs: fedora-rawhide: runs-on: ubuntu-latest + timeout-minutes: 45 strategy: fail-fast: false @@ -130,6 +131,7 @@ jobs: alpine-edge: runs-on: ubuntu-latest + timeout-minutes: 45 strategy: fail-fast: false diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 0e3cd85de..3a41e92cf 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -33,6 +33,14 @@ env: # stable across patch bumps. OPENSSL4_VERSION: 4.0.2 +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: # Single source of truth for the test matrix. The version-tested language # modules (go/java/node/php/python/ruby) are derived from pkg/eol.json; the @@ -41,6 +49,7 @@ jobs: # and the "minimal" image — those are Docker packaging flavors, not test axes. prepare: runs-on: ubuntu-latest + timeout-minutes: 10 outputs: builds: ${{ steps.list.outputs.builds }} steps: @@ -60,6 +69,7 @@ jobs: test: needs: prepare runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -530,6 +540,7 @@ jobs: # privilege-sensitive cases (clone creds) work. c-tests: runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v5 @@ -562,6 +573,7 @@ jobs: # nothing above the TLS layer changes with the OpenSSL version. openssl4: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index ac66d0164..afaded0ec 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -31,6 +31,7 @@ jobs: (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')) && contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.issue.author_association)) runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read pull-requests: read @@ -86,6 +87,7 @@ jobs: (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')) && !contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.issue.author_association)) runs-on: ubuntu-latest + timeout-minutes: 5 permissions: issues: write pull-requests: write diff --git a/.github/workflows/eol-check.yml b/.github/workflows/eol-check.yml index ba28091c8..0484ac4ca 100644 --- a/.github/workflows/eol-check.yml +++ b/.github/workflows/eol-check.yml @@ -27,9 +27,18 @@ on: permissions: contents: read +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: eol-check: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read issues: write # scheduled run may open/update a tracking issue diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index 34e2a1914..8a3425598 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -11,6 +11,7 @@ permissions: {} jobs: Fuzzing: runs-on: ubuntu-latest + timeout-minutes: 60 permissions: security-events: write steps: diff --git a/.github/workflows/lint-whitespace.yml b/.github/workflows/lint-whitespace.yml index beb805b0c..c85ae416e 100644 --- a/.github/workflows/lint-whitespace.yml +++ b/.github/workflows/lint-whitespace.yml @@ -7,9 +7,18 @@ on: pull_request: types: [ opened, synchronize ] +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: check-whitespace: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v5 with: diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 34519b2d2..33c705cc3 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -51,6 +51,7 @@ jobs: # consumed by both `build` and `merge` via fromJSON(). setup: runs-on: ubuntu-latest + timeout-minutes: 10 outputs: version: ${{ steps.meta.outputs.version }} variants: ${{ steps.meta.outputs.variants }} @@ -84,6 +85,7 @@ jobs: build: needs: setup runs-on: ${{ matrix.arch.runner }} + timeout-minutes: 90 permissions: contents: read packages: write @@ -146,6 +148,7 @@ jobs: merge: needs: [setup, build] runs-on: ubuntu-latest + timeout-minutes: 20 permissions: contents: read packages: write diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 2a5aee3b1..0ff58a9b0 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -53,10 +53,19 @@ on: - '.github/scripts/check-sanitizer-reports.sh' - '.github/sanitizer-allowlist.txt' +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: sanitize: name: "sanitize (${{ matrix.runtime }})" runs-on: ubuntu-latest + timeout-minutes: 40 # NOTE: `continue-on-error` is deliberately NOT set at job level here. # It is set on every step an experimental leg can fail in: the diff --git a/.github/workflows/security-seccomp.yml b/.github/workflows/security-seccomp.yml index e4c640b52..f9dc377bd 100644 --- a/.github/workflows/security-seccomp.yml +++ b/.github/workflows/security-seccomp.yml @@ -18,10 +18,19 @@ on: permissions: contents: read +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: seccomp-af-alg: name: Verify AF_ALG seccomp mitigation runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/unitctl.yml b/.github/workflows/unitctl.yml index a0f35b57f..1c74c2391 100644 --- a/.github/workflows/unitctl.yml +++ b/.github/workflows/unitctl.yml @@ -21,9 +21,18 @@ on: permissions: contents: write +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: test: runs-on: ${{ matrix.os }} + timeout-minutes: 25 defaults: run: working-directory: tools/unitctl @@ -74,6 +83,7 @@ jobs: build: runs-on: ${{ matrix.os }} + timeout-minutes: 30 defaults: run: working-directory: tools/unitctl @@ -177,6 +187,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' needs: [build] runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/download-artifact@v8 with: From ec20d6af33f86c7d48417ad64ad41d6b02a97f5c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 28 Aug 2026 22:51:52 +0200 Subject: [PATCH 063/391] router: release the start slot when a start attempt fails early nxt_router_start_app_process_handler() is entered owning one app->pending_processes increment: the initiator takes it under the app mutex, and nxt_router_start_app_process() rolls it back only when nxt_port_post() fails -- that is, only when the handler never ran. Once the handler runs, it must dispose of that reference in one of three ways: transfer it to an armed RPC, which nxt_router_app_port_ready() or nxt_router_app_port_error() releases; transfer it to the prototype wait queue, which replays it as a fresh invocation; or release it when the attempt fails. The third leg was missing, so all three goto failed sites -- a buffer allocation, an RPC registration, and a port write -- leaked the slot. Nothing recovers it. Requests never own the counter, so no timeout path returns it, and the leak is not visible as an underflow. It gates new starts through nxt_router_app_can_start(), and max_pending_processes is spare_processes or 1, so with the default spare: 0 a single failure stops the application starting workers for the life of the router. The failure paths are allocation and IPC failures, reachable exactly when an application most needs to scale. Release the slot at the failed: label rather than at the three sites: the label is where the reference goes out of scope, so it also covers entry paths added later, including prototype replays. A pre-arm failure is the same event nxt_router_app_port_error() handles, only earlier, so the two share one helper: the requests waiting on the attempt are failed rather than left to time out, which is what port_error already did. The release cannot double-count. nxt_port_rpc_cancel() deletes the registration without invoking the error handler, so no path from failed: also reaches port_error. Two neighbouring windows on the same counter are deliberately left open. A refused port queue strands an already-armed start, which needs port provenance to fix and is tracked in #223 -- the shared helper here is the primitive that work completes into. And a failed prototype start leaves proto_port_requests set, parking every later attempt on a prototype that is not coming; that is a distinct defect in port_error and gets its own change. All three sites were also silent outside --debug: the slot leaks upward instead of underflowing, so the sibling nxt_asserts cannot see it, and nothing logged. The label now alerts, naming the application, so an operator can tell a wedged application from a merely busy one. Two tests cover the label. nxt_router_start_fail_test.c drives the handler once through the port-write site, failing the port layer's message allocation with the existing nxt_port_test_msg_alloc_failures() hook. nxt_router_start_fail_soak_test.c loops 1000 early failures through all three sites, cycling them and requiring app->pending_processes back at its baseline after every one; the baselines alternate 0 and 2, so an underflow shows up as a miscount and not only as a uint32 wrap, which keeps the check meaningful in a build with nxt_assert compiled out. Afterwards the application must still pass the start gate, and one successful start must park exactly one increment on the armed RPC for the router's own error handler to give back. Closes #214. Co-Authored-By: Claude Opus 5 --- auto/sources | 2 + src/nxt_router.c | 36 ++- src/nxt_router.h | 4 + src/test/nxt_router_start_fail_soak_test.c | 332 +++++++++++++++++++++ src/test/nxt_router_start_fail_test.c | 171 +++++++++++ src/test/nxt_tests.c | 8 + src/test/nxt_tests.h | 2 + 7 files changed, 552 insertions(+), 3 deletions(-) create mode 100644 src/test/nxt_router_start_fail_soak_test.c create mode 100644 src/test/nxt_router_start_fail_test.c diff --git a/auto/sources b/auto/sources index 75ef9930d..6d648eda6 100644 --- a/auto/sources +++ b/auto/sources @@ -179,6 +179,8 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_mmap_range_test.c \ src/test/nxt_port_ready_test.c \ src/test/nxt_router_new_port_test.c \ + src/test/nxt_router_start_fail_test.c \ + src/test/nxt_router_start_fail_soak_test.c \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_fd_test.c \ " diff --git a/src/nxt_router.c b/src/nxt_router.c index 58488c3f3..24106a090 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -229,6 +229,7 @@ static void nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static void nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); +static void nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app); static void nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i); static void nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app); @@ -395,7 +396,7 @@ nxt_router_greet_controller(nxt_task_t *task, nxt_port_t *controller_port) } -static void +void nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, void *data) { @@ -484,12 +485,26 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, nxt_router_app_joint_use(task, app->joint, 1); + goto skip; + failed: + /* + * The attempt is over and no handler remains armed for it -- the two + * earlier sites never armed one, and the third cancels its registration + * with nxt_port_rpc_cancel() -- so nothing will ever report it back. + * Release the pending_processes slot the initiator took, exactly as + * nxt_router_app_port_error() does for the attempts that do get that far. + */ + + nxt_alert(task, "app '%V' failed to start a process", &app->name); + if (b != NULL) { nxt_mp_free(b->data, b); } + nxt_router_app_start_failed(task, app); + skip: nxt_router_app_use(task, app, -1); @@ -5041,8 +5056,6 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, { nxt_app_t *app; nxt_app_joint_t *app_joint; - nxt_queue_link_t *link; - nxt_http_request_t *r; nxt_app_joint_rpc_t *app_joint_rpc; nxt_assert(data != NULL); @@ -5064,6 +5077,23 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_debug(task, "app '%V' %p start error", &app->name, app); + nxt_router_app_start_failed(task, app); +} + + +/* + * A start attempt that will never yield a port: give back the + * pending_processes slot its initiator took and, if the application is left + * with no way to answer at all, fail the requests waiting for an + * acknowledgement instead of letting them sit until they time out. + */ + +static void +nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app) +{ + nxt_queue_link_t *link; + nxt_http_request_t *r; + link = NULL; nxt_thread_mutex_lock(&app->mutex); diff --git a/src/nxt_router.h b/src/nxt_router.h index 8ca5dbbea..d9d60d1db 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -265,6 +265,10 @@ void nxt_router_access_log_release(nxt_task_t *task, /* Not static so the descriptor-ownership test can drive it directly. */ void nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +/* Not static so the start-failure test can drive it directly. */ +void nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, + void *data); + void nxt_router_access_log_reopen_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/src/test/nxt_router_start_fail_soak_test.c b/src/test/nxt_router_start_fail_soak_test.c new file mode 100644 index 000000000..57c83231f --- /dev/null +++ b/src/test/nxt_router_start_fail_soak_test.c @@ -0,0 +1,332 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Failure-injection soak for issue #214. The single-shot test proves that + * one early failure gives the pending_processes slot back; this one proves + * the accounting is exact under repetition and reaches all three "goto + * failed" sites in nxt_router_start_app_process_handler(): + * + * site 1 the prototype buffer allocation, forced by an app conf length + * no allocator can satisfy; + * site 2 nxt_port_rpc_register_handler_ex(), forced by the NXT_TESTS + * rpc allocation hook; + * site 3 nxt_port_socket_write2(), forced by the NXT_TESTS port message + * allocation hook (the site the single-shot test uses). + * + * Every iteration seeds the counter at a baseline, adds the one increment + * the initiator takes, and requires it back at the baseline afterwards -- + * a leak leaves baseline+1, a double release leaves baseline-1 (or wraps + * to UINT32_MAX at baseline 0). Both are caught without nxt_assert, so + * the check survives a non-debug build. Baselines alternate between 0 and + * a non-zero value so an underflow is caught as an ordinary miscount and + * not only as a wrap. + * + * After the loop the application must still be startable -- that is the + * damage #214 does -- and one successful start must still park exactly one + * increment on the RPC for nxt_router_app_port_error() to release. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_ROUTER_START_FAIL_SOAK_N 1000 + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_start_fail_soak_test_app; + + +/* + * Mirrors the nxt_inline nxt_router_app_can_start() predicate in + * src/nxt_router.c:1218. Being nxt_inline, the gate #214 wedges is private + * to the router's translation unit and cannot be called from here, so this + * copy must be kept in sync with it. + */ + +static nxt_bool_t +nxt_router_start_fail_soak_can_start(nxt_app_t *app) +{ + return app->processes + app->pending_processes < app->max_processes + && app->pending_processes < app->max_pending_processes; +} + + +nxt_int_t +nxt_router_start_fail_soak_test(nxt_thread_t *thr) +{ + uint32_t baseline; + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_bool_t app_mutex; + nxt_uint_t i, site; + nxt_task_t *task; + nxt_port_t *router_port, *dport; + nxt_runtime_t *rt, *saved_rt; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test started"); + + ret = NXT_ERROR; + router_port = NULL; + dport = NULL; + joint = NULL; + app_mutex = 0; + app = &nxt_router_start_fail_soak_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Site 1 allocates from task->thread->engine->mem_pool, so unlike the + * single-shot fixture this one needs an engine. Only the members the + * reached code touches are set; see src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The destination port. No queue and write_ready unset, so + * nxt_port_socket_write2() takes the nxt_port_msg_chk_insert() path: + * the test hook fails its allocation for site 3, and without the hook + * the message is simply queued and the write succeeds. + */ + + dport = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(dport == NULL)) { + goto done; + } + + dport->pair[0] = -1; + dport->pair[1] = -1; + dport->socket.fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "start-fail-soak-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + app->max_processes = 8; + app->max_pending_processes = 1; + + for (i = 0; i < NXT_ROUTER_START_FAIL_SOAK_N; i++) { + site = i % 3; + baseline = (i % 2) ? 2 : 0; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = baseline + 1; + app->use_count = 2; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + app->proto_port_requests = 0; + + if (site == 0) { + /* + * Site 1: no prototype, so the handler builds the START_PROCESS + * payload first. A length no allocator can serve makes + * nxt_buf_mem_alloc() return NULL before the buffer is written + * to, so app->conf may stay empty. + */ + app->proto_port = NULL; + app->conf.length = (size_t) 1 << 46; + + } else { + app->proto_port = dport; + app->conf.length = 0; + + if (site == 1) { + nxt_port_rpc_test_alloc_failures(1); + + } else { + nxt_port_test_msg_alloc_failures(1); + } + } + + nxt_router_start_app_process_handler(task, router_port, app); + + nxt_port_rpc_test_alloc_failures(0); + nxt_port_test_msg_alloc_failures(0); + + if (app->pending_processes != baseline) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: iteration %ui site %ui " + "pending_processes %uD (expected %uD)", + i, site + 1, app->pending_processes, baseline); + goto done; + } + + if (app->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: iteration %ui site %ui " + "app use_count %d (expected 1)", + i, site + 1, (int) app->use_count); + goto done; + } + + if (joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: iteration %ui site %ui " + "joint use_count %uD (expected 1)", + i, site + 1, joint->use_count); + goto done; + } + } + + /* The gate #214 wedges: after the failures the app must still start. */ + + app->processes = 0; + app->pending_processes = 0; + app->conf.length = 0; + app->proto_port = dport; + + if (!nxt_router_start_fail_soak_can_start(app)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: app cannot start after " + "%d failures, processes %uD pending %uD", + NXT_ROUTER_START_FAIL_SOAK_N, app->processes, + app->pending_processes); + goto done; + } + + /* + * A start that succeeds parks the initiator's increment on the armed + * RPC and takes a joint reference; nothing is released here. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->pending_processes != 1 || joint->use_count != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: successful start left " + "pending %uD joint %uD (expected 1 and 2)", + app->pending_processes, joint->use_count); + goto done; + } + + /* + * Tearing the port's registrations down drives the router's own error + * handler for the armed RPC, which is how a start that never answers is + * reported; it must give the parked increment back. + */ + + nxt_port_rpc_close(task, router_port); + + if (app->pending_processes != 0 || joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: release left pending %uD " + "joint %uD (expected 0 and 1)", + app->pending_processes, joint->use_count); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test passed: %d iterations over " + "failure sites 1-3", + NXT_ROUTER_START_FAIL_SOAK_N); + + ret = NXT_OK; + +done: + + if (dport != NULL) { + nxt_port_use(task, dport, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_router_start_fail_test.c b/src/test/nxt_router_start_fail_test.c new file mode 100644 index 000000000..72c362982 --- /dev/null +++ b/src/test/nxt_router_start_fail_test.c @@ -0,0 +1,171 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for issue #214: nxt_router_start_app_process_handler() + * is entered owning one app->pending_processes increment (its initiator + * took it before posting the work), and its failure paths used to return + * without giving it back. The counter gates every future start + * (nxt_router_app_can_start()), so each leak permanently costs the + * application one start slot; with the default max_pending_processes of 1 + * a single failure wedges the application for the router's lifetime. + * + * The deterministic failure used here is the nxt_port_socket_write2() + * site: the NXT_TESTS allocation-failure hook makes the port layer's + * message allocation fail, the write returns NXT_ERROR, and the handler + * takes "goto failed" after cancelling the RPC registration it had just + * made. On entry the app is set up the way a real start attempt finds + * it: pending_processes already counted, one extra app use for the posted + * work item. + * + * This single-shot test reaches that third site only. All three failure + * sites, and the accounting across repeated failures, are covered by + * src/test/nxt_router_start_fail_soak_test.c. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* + * File scope rather than a stack object: the handler hands the application to + * the router's reference-counting helpers, so its storage must outlive the + * frame even though this test's arrangement never lets it be posted anywhere. + */ +static nxt_app_t nxt_router_start_fail_test_app; + + +nxt_int_t +nxt_router_start_fail_test(nxt_thread_t *thr) +{ + nxt_app_t *app; + nxt_int_t ret; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *dport; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router start fail test started"); + + ret = NXT_ERROR; + router_port = NULL; + dport = NULL; + app_mutex = 0; + app = &nxt_router_start_fail_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + return NXT_ERROR; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The destination port the START_PROCESS message is written to. No + * queue and write_ready unset, so nxt_port_socket_write2() takes the + * nxt_port_msg_chk_insert() path, whose allocation the test hook + * fails. + */ + + dport = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(dport == NULL)) { + goto done; + } + + dport->pair[0] = -1; + dport->pair[1] = -1; + dport->socket.fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "start-fail-test"); + + /* + * A prototype port makes the handler take the plain app-start branch: + * no buffer allocation, straight to the RPC registration and the + * write. + */ + app->proto_port = dport; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = 1; + app->use_count = 2; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + nxt_port_test_msg_alloc_failures(1); + + nxt_router_start_app_process_handler(task, router_port, app); + + nxt_port_test_msg_alloc_failures(0); + + if (app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail test: pending_processes %uD leaked " + "(expected 0)", app->pending_processes); + goto done; + } + + if (app->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail test: app use_count %d (expected 1)", + (int) app->use_count); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router start fail test passed"); + + ret = NXT_OK; + +done: + + if (dport != NULL) { + nxt_port_use(task, dport, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 7b256d8b7..3065670c0 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -201,6 +201,14 @@ main(int argc, char **argv) return 1; } + if (nxt_router_start_fail_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_router_start_fail_soak_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 7206e1037..660ef1f00 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -74,6 +74,8 @@ nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); +nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); +nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); From 9f45865359d1e7e75e5e3759e660d2239b00da4a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 07:55:02 +0200 Subject: [PATCH 064/391] test(port): iterate control messages through NXT_CMSG_NXTHDR musl's CMSG_NXTHDR expands to a comparison between a size_t and the ptrdiff_t of a pointer subtraction, so clang rejects it under -Werror with -Wsign-compare. nxt_socket_msg.h already carries NXT_CMSG_NXTHDR for exactly this, and both production call sites use it; this test reached for the raw macro and so failed to build on musl with clang. Alpine's aports CI found it, building 1.36.1 with clang22: src/test/nxt_unit_port_recv_test.c:107:17: error: comparison of integers of different signs: '__size_t' and '__ptrdiff_t' The repository's own alpine-edge clang leg covers this configuration, but build-distros.yml is workflow_dispatch-only and has never run. Co-Authored-By: Claude Opus 5 --- src/test/nxt_unit_port_recv_test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/nxt_unit_port_recv_test.c b/src/test/nxt_unit_port_recv_test.c index d93eb12cd..c798177d1 100644 --- a/src/test/nxt_unit_port_recv_test.c +++ b/src/test/nxt_unit_port_recv_test.c @@ -31,6 +31,7 @@ */ #include "nxt_main.h" +#include "nxt_socket_msg.h" #include "nxt_port.h" #include "nxt_port_queue.h" #include "nxt_app_queue.h" @@ -104,7 +105,7 @@ nxt_port_recv_test_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; - cmsg = CMSG_NXTHDR(&msg, cmsg)) + cmsg = NXT_CMSG_NXTHDR(&msg, cmsg)) { size = cmsg->cmsg_len - CMSG_LEN(0); From 630616c313c5404f14b5631404e9d2d37f29c984 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 12:45:51 +0200 Subject: [PATCH 065/391] build(wasm): update bindgen so the module builds under clang 22 bindgen 0.68.1 dates from 2023 and cannot parse Unit's headers with clang 22. It does not fail: it emits opaque placeholder structs whose only field is `_address`, and the crate then fails to compile with about twenty errors of the form "no field `request` on type `nxt_unit_request_info_s`" -- pointing at Unit's own code rather than at the parse that never happened. That is not a hypothetical toolchain. Alpine defaults `clang-dev` to clang 22 in both edge and the current stable 3.24, so the module cannot be packaged there today, and every distribution moving to clang 22 meets the same wall. Update to bindgen 0.72 and construct the callbacks explicitly: CargoCallbacks became a struct requiring ::new() in 0.69. The lockfile also drops seven transitive dependencies (home, lazycell, lazy_static, linux-raw-sys, peeking_take_while, rustc-hash, rustix, which) and gains itertools. Verified that the new bindings describe the same memory layout as the old, rather than merely compiling: - clang 18 + bindgen 0.68.1 and clang 22 + bindgen 0.72.1 produce byte-identical declarations -- 1,346 lines covering 154 nxt_ structs with their #[repr] attributes attached, md5 16db8308. - With layout_tests enabled, 0.68.1 emits 170 assertions computed by the C compiler from Unit's headers; all 170 pass. - Those of the 147 that cover nxt_ types were compiled against the 0.72.1 bindings and run: 147 passed, 0 failed. So the new bindings are checked directly against the C layout, not inferred from a diff. - Both artifacts were confirmed by their generator stamp to come from the bindgen the run asked for. Also built on glibc with clang 19 (rust:1.98): module links, 154 nxt_ structs, no opaque placeholders. Co-Authored-By: Claude Opus 5 --- src/wasm-wasi-component/Cargo.lock | 108 +++++++---------------------- src/wasm-wasi-component/Cargo.toml | 2 +- src/wasm-wasi-component/build.rs | 2 +- 3 files changed, 27 insertions(+), 85 deletions(-) diff --git a/src/wasm-wasi-component/Cargo.lock b/src/wasm-wasi-component/Cargo.lock index 818bb2077..e219eaece 100644 --- a/src/wasm-wasi-component/Cargo.lock +++ b/src/wasm-wasi-component/Cargo.lock @@ -63,25 +63,22 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools 0.13.0", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", + "rustc-hash", "shlex 1.3.0", "syn", - "which", ] [[package]] @@ -138,7 +135,7 @@ dependencies = [ "io-lifetimes 3.0.1", "ipnet", "maybe-owned", - "rustix 1.1.4", + "rustix", "rustix-linux-procfs", "windows-sys 0.61.2", "winx", @@ -153,7 +150,7 @@ dependencies = [ "cap-primitives", "io-extras", "io-lifetimes 3.0.1", - "rustix 1.1.4", + "rustix", ] [[package]] @@ -300,7 +297,7 @@ dependencies = [ "postcard", "pulley-interpreter", "regalloc2", - "rustc-hash 2.1.3", + "rustc-hash", "serde", "serde_derive", "sha2", @@ -502,7 +499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes 2.0.4", - "rustix 1.1.4", + "rustix", "windows-sys 0.59.0", ] @@ -676,15 +673,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "http" version = "1.4.2" @@ -901,6 +889,15 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -916,18 +913,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128" version = "0.2.7" @@ -962,12 +947,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1010,7 +989,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" dependencies = [ - "rustix 1.1.4", + "rustix", ] [[package]] @@ -1058,12 +1037,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1181,7 +1154,7 @@ dependencies = [ "bumpalo", "hashbrown 0.17.1", "log", - "rustc-hash 2.1.3", + "rustc-hash", "serde", "smallvec", ] @@ -1221,31 +1194,12 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" @@ -1255,7 +1209,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] @@ -1266,7 +1220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" dependencies = [ "once_cell", - "rustix 1.1.4", + "rustix", ] [[package]] @@ -1654,7 +1608,7 @@ dependencies = [ "once_cell", "postcard", "pulley-interpreter", - "rustix 1.1.4", + "rustix", "semver", "serde", "serde_derive", @@ -1751,7 +1705,7 @@ dependencies = [ "cranelift-frontend", "cranelift-native", "gimli", - "itertools", + "itertools 0.14.0", "log", "object", "pulley-interpreter", @@ -1774,7 +1728,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "rustix 1.1.4", + "rustix", "wasmtime-environ", "wasmtime-internal-versioned-export-macros", "windows-sys 0.61.2", @@ -1854,7 +1808,7 @@ dependencies = [ "futures", "io-lifetimes 3.0.1", "rand", - "rustix 1.1.4", + "rustix", "thiserror 2.0.18", "tokio", "tracing", @@ -1907,18 +1861,6 @@ dependencies = [ "leb128", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "wiggle" version = "47.0.2" diff --git a/src/wasm-wasi-component/Cargo.toml b/src/wasm-wasi-component/Cargo.toml index 82de81083..1bde3e10e 100644 --- a/src/wasm-wasi-component/Cargo.toml +++ b/src/wasm-wasi-component/Cargo.toml @@ -27,7 +27,7 @@ wasmtime-wasi = "47.0.2" wasmtime-wasi-http = { version = "47.0.2", default-features = false, features = ['p2'] } [build-dependencies] -bindgen = "0.68.1" +bindgen = "0.72" cc = "1.0.83" [profile.dev] diff --git a/src/wasm-wasi-component/build.rs b/src/wasm-wasi-component/build.rs index 5ea74f172..de04ec9a5 100644 --- a/src/wasm-wasi-component/build.rs +++ b/src/wasm-wasi-component/build.rs @@ -15,7 +15,7 @@ fn main() { .blocklist_function("nxt_vsprintf") // Tell cargo to invalidate the built crate whenever any of the // included header files changed. - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) // disable some features which aren't necessary .layout_tests(false) .derive_debug(false) From d8feac333225e2367d6bc48a2ac11bfac6279a4b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 15:20:20 +0200 Subject: [PATCH 066/391] ci(distros): fix the PHP option, and run rolling distros weekly The Alpine leg has been failing on every run at ./configure: error: invalid PHP option "--php-config=/usr/bin/php-config83" because the PHP module's option is `--config=`, not `--php-config=` (auto/modules/php:17). The workflow is manual-only, so nobody was watching it fail; the most recent run is red on exactly this. Give it a weekly schedule rather than the pull request trigger the commented-out block proposed. Fedora rawhide and Alpine edge move between our pull requests, so their breakage arrives on the distribution's schedule and not on ours: as a PR gate they would add roughly 25 job-minutes to each of ~107 weekly runs to catch bugs that were not in the pull request, into a queue that has already made a 15-minute run take an hour. Weekly costs about 30 job-minutes in total. A red schedule nobody caused is a check nobody reads, so a scheduled failure opens or updates a tracking issue instead, the same pattern eol-check.yml already uses for its weekly run. These two legs are the only place the project meets a rolling toolchain: Fedora rawhide currently carries clang 22.1.8, which is the compiler that exposed the bindgen parse failure in the wasm module. Neither leg builds that module yet, so this repairs and schedules them first; adding wasm coverage is worth doing separately, once it can be tested rather than assumed. Co-Authored-By: Claude Opus 5 --- .github/workflows/build-distros.yml | 72 +++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 7055ee19d..3e6863b38 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -1,23 +1,17 @@ name: "Build (Fedora/Alpine · GCC/Clang)" on: - workflow_dispatch: # manual only -# push: -# branches: master -# paths: -# - configure -# - 'auto/**' -# - 'src/**' -# - 'test/**' -# - '.github/workflows/build-distros.yml' -# pull_request: -# branches: master -# paths: -# - configure -# - 'auto/**' -# - 'src/**' -# - 'test/**' -# - '.github/workflows/build-distros.yml' + # Rolling distributions break on their own schedule, not on ours, so this is + # a weekly gate rather than a PR gate: Fedora rawhide and Alpine edge move + # under us between pull requests, and a red check nobody caused is a check + # nobody reads. A failure opens or updates a tracking issue instead. + schedule: + - cron: '17 4 * * 1' + workflow_dispatch: + +permissions: + contents: read + issues: write jobs: @@ -169,7 +163,7 @@ jobs: - name: configure unit-php run: | ln -s /usr/lib/libphp83.so /usr/lib/libphp.so - ./configure php --php-config=/usr/bin/php-config83 + ./configure php --config=/usr/bin/php-config83 - name: make unit-php run: make -j 4 @@ -196,4 +190,44 @@ jobs: run: ./configure java - name: make unit-java - run: make -j 4 java \ No newline at end of file + run: make -j 4 java + + # A weekly red check that nobody caused is a check nobody reads, so a + # scheduled failure is reported as a tracking issue instead, the same way + # eol-check.yml handles its weekly run. + report: + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: [ fedora-rawhide, alpine-edge ] + if: failure() && github.event_name == 'schedule' + + steps: + - name: Open or update tracking issue + env: + GH_TOKEN: ${{ github.token }} + # This job does not check out the repository, so gh has no local git + # remote to infer the target from and would exit before creating the + # issue -- silently defeating the point of the job. + GH_REPO: ${{ github.repository }} + run: | + title='Rolling distro build is failing (Fedora rawhide / Alpine edge)' + { + echo 'The weekly rolling-distribution build failed. These distributions' + echo 'move between our pull requests, so a failure here is usually a' + echo 'toolchain or package change rather than a regression in Unit.' + echo + echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo + echo 'Reproduce locally, e.g. for Alpine:' + echo + echo '```' + echo 'docker run --rm -it --platform linux/amd64 alpine:edge sh' + echo '```' + } > issue-body.md + existing=$(gh issue list --state open --search "in:title ${title}" \ + --json number,title --jq ".[] | select(.title == \"${title}\") | .number" | head -n1) + if [ -n "${existing}" ]; then + gh issue comment "${existing}" --body-file issue-body.md + else + gh issue create --title "${title}" --body-file issue-body.md + fi From a8c9318e3efb832a05b6b5d4a7196ce3268da0a4 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 29 Aug 2026 18:00:10 +0200 Subject: [PATCH 067/391] ci(distros): build the wasm component, so the rolling gate catches bindgen This workflow is the only place the project meets a rolling toolchain, and until now it did not build the one module that a rolling toolchain breaks. Fedora rawhide already carries clang 22.1.8 -- the exact compiler behind #250, where bindgen 0.68.1 could not parse Unit's headers -- and it sat unused: the fedora leg built only the C wasm module, and the alpine leg stopped at java. The failure this gate is for does not announce itself. bindgen does not error on a libclang it cannot parse with; it emits opaque `_address` placeholder structs, and the crate then fails to compile with twenty errors that all point at Unit's own code. Nothing short of building the component finds that. Build it on the gcc leg only. bindgen goes through libclang whatever the C compiler is, so the clang leg would parse the same headers with the same libclang and cost twice as much for it. Both legs verified in their real containers before this was written, which is how the alpine half came to include linux-headers: bindgen parses `linux/openat2.h` via nxt_unix.h, a header the C build never reaches, so the leg failed on a missing include rather than on anything to do with clang. Fedora needed only clang-devel. Results: fedora rawhide, clang 22.1.8, bindgen 0.72.1, 154 nxt_ structs, no opaque placeholders, module 21,953,336 bytes; alpine edge the same at 21,855,008 bytes. Co-Authored-By: Claude Opus 5 --- .github/workflows/build-distros.yml | 33 +++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 3e6863b38..214693519 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -37,7 +37,7 @@ jobs: python-unversioned-command python3 python3-devel \ php-devel php-embedded perl-devel perl-ExtUtils-Embed \ ruby-devel java-devel nodejs-devel nodejs-npm golang \ - cargo rust cmake + cargo rust cmake clang-devel if [ "${{ matrix.compiler }}" = "clang" ]; then dnf -y install --setopt=install_weak_deps=False clang fi @@ -123,6 +123,22 @@ jobs: - name: make unit-wasm run: make wasm + # The Rust component module is the reason this workflow is worth running + # weekly: bindgen parses Unit's headers with libclang, and a new libclang + # breaks that silently -- it emits opaque `_address` placeholder structs + # rather than failing, so the crate then fails to compile with errors that + # point at Unit's code. That is what #250 fixed for clang 22, which this + # distribution already ships. Only the gcc leg builds it: bindgen uses + # libclang whatever the C compiler is, so the clang leg would test the + # same parse twice. + - name: configure unit-wasm-wasi-component + run: ./configure wasm-wasi-component + if: matrix.compiler == 'gcc' + + - name: make unit-wasm-wasi-component + run: make wasm-wasi-component + if: matrix.compiler == 'gcc' + alpine-edge: runs-on: ubuntu-latest timeout-minutes: 45 @@ -142,7 +158,7 @@ jobs: apk add gcc make musl-dev openssl-dev pcre2-dev curl \ zlib-dev zstd-dev brotli-dev \ php83-dev php83-embed python3-dev perl-dev ruby-dev \ - openjdk21-jdk cargo rust + openjdk21-jdk cargo rust clang-dev linux-headers if [ "${{ matrix.compiler }}" = "clang" ]; then apk add clang fi @@ -192,6 +208,19 @@ jobs: - name: make unit-java run: make -j 4 java + # See the note on the same steps in fedora-rawhide: this is the gate that + # would have caught the bindgen/libclang breakage #250 fixed. Alpine + # matters twice over here -- it defaults clang-dev to clang 22 in edge AND + # in stable 3.24, and it is where the module could not be packaged at all + # before #250. gcc leg only: bindgen goes through libclang either way. + - name: configure unit-wasm-wasi-component + run: ./configure wasm-wasi-component + if: matrix.compiler == 'gcc' + + - name: make unit-wasm-wasi-component + run: make wasm-wasi-component + if: matrix.compiler == 'gcc' + # A weekly red check that nobody caused is a check nobody reads, so a # scheduled failure is reported as a tracking issue instead, the same way # eol-check.yml handles its weekly run. From 4c79840d477dc4779e66c6cd287d1354c51edaff Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 16:44:32 +0200 Subject: [PATCH 068/391] port: authenticate PROCESS_READY by the sender's kernel pid nxt_port_process_ready_handler() trusted msg->port_msg.pid, a value the sender writes into the message itself, and acted on the process it named: it marked that process ready and re-pointed its port at the shared memory queue the message carried. Nothing checked who sent it. Every application worker can send that message to a process that installs this handler. A port socket is per-owner and its pair[1] is duplicated into every peer that keeps it (nxt_proc_keep_matrix[]), so a worker holds its prototype's write end -- and an embedded module holds main's. nxt_port_handler() dispatches on the wire type alone. So one compromised worker can announce a sibling and hand main or the prototype a queue descriptor of its own making; the receiver installs it on the sibling's port and then broadcasts that port to every peer. From there the attacker reads and writes another application's port traffic. The same forgery can mark a process ready that never started, and re-point a port that is already live and serving. Reject a PROCESS_READY whose kernel-validated sender pid is not the pid of the process it names, close the descriptors it carried, and log the two pids. The check runs immediately after the lookup, before the state is read or written, so a refused message leaves no trace -- the same rule the empty-ports guard already follows. The credential is compared against process->isolated_pid, and only against it. isolated_pid is by construction the pid of that process as this receiver sees it: it is the fork() return value stored by nxt_process_create(), and nxt_proto_process_created_handler() leaves it alone when it rewrites process->pid to the global pid of a pid-isolated worker. The lookup still has to use msg->port_msg.pid, because the runtime hash is keyed on the global pid, but the authentication does not. Accepting process->pid as well -- the shape a first draft of this would naturally take -- is a bypass, not a convenience. In a pid namespace the credential the prototype reads is the namespace-local pid, so a task whose namespace-local pid happens to equal the victim's global pid would pass, and a worker that can fork can walk towards such a collision. A non-positive isolated_pid is refused rather than compared. A process record that no fork() in this process created keeps it at 0 -- nxt_runtime_process_get() fills only ->pid, which is how main learns of an application worker through the WHOAMI exchange -- and a credential of 0, or the -1 that nxt_port_queue_read_handler() leaves in a message dequeued from shared memory, would otherwise match by accident. No legitimate PROCESS_READY is lost: a worker announces itself to its parent prototype, and every child of main is forked by main. PR 2 of #231 will fail the start of a worker whose queue is refused, and inherits an explicit guard here rather than an implicit one. A repeated PROCESS_READY from the authenticated owner stays tolerated: the gate is what now refuses a forged repeat, and a genuine one has to keep working, so the concession that used to cover both no longer has to cover the forgery. The rationale comments that said this handler cannot be authenticated are rewritten rather than left to invite a revert, and the file's sender-ACL TODO no longer cites SO_PASSCRED as a missing precondition: nxt_socketpair_create() has set it on both ends for some time, NXT_HAVE_MSGHDR_CMSGCRED platforms get an SCM_CREDS control message from nxt_socket_msg_oob_init(), and both receive paths fill msg->cmsg_pid. The gate is compiled only where the platform hands the receiver a kernel-validated pid (NXT_USE_CMSG_PID: Linux SCM_CREDENTIALS or BSD SCM_CREDS). Elsewhere -- macOS, NetBSD, OpenBSD, illumos -- PROCESS_READY stays as unauthenticated as it was, exactly like every sibling gate in nxt_main_process.c, nxt_cert.c and nxt_script.c. The check is not written against nxt_recv_msg_cmsg_pid()'s fallback either: that fallback is msg->port_msg.pid, which for a pid-isolated worker is the global pid and would fail against isolated_pid. src/test/nxt_port_ready_test.c covers it. The fixture now models a pid-isolated process -- process->pid and process->isolated_pid deliberately differ -- so a case cannot pass whichever of the two the handler compared; every pre-existing case sets msg.cmsg_pid to the isolated pid, which is what the handler now requires of them. Three new cases assert that a foreign credential, the process's own global pid, and a credential matching an unforked record's isolated_pid of 0 are each refused: the descriptors are closed, the state is unchanged and the port keeps its queue. The last of them runs after a queue is installed and the process is ready, which is the state where an accepted message would do the damage. The rejected messages carry a valid, mappable queue descriptor, so a handler without the gate is caught installing it. Each case was shown to fail without the code it tests: with the gate removed the foreign-sender case fails, with process->pid accepted as well the global-pid case fails, and without the isolated_pid <= 0 guard the unforked-record case fails. Co-Authored-By: Claude Opus 5 --- src/nxt_port.c | 118 +++++++++++++------ src/test/nxt_port_ready_test.c | 204 ++++++++++++++++++++++++++++++++- 2 files changed, 283 insertions(+), 39 deletions(-) diff --git a/src/nxt_port.c b/src/nxt_port.c index e1abd394a..7557f1996 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -241,15 +241,27 @@ nxt_port_enable(nxt_task_t *task, nxt_port_t *port, * * The audit recommends rejecting application workers / prototypes * that forge cert/script/socket/access-log/etc. messages to the - * main process. A first draft was implemented but caused - * regressions in the isolation test suite — the kernel-validated - * sender PID (SCM_CREDENTIALS) is not consistently available on - * the existing socketpair setup, and the message-dispatch path - * runs before the prototype's process registration completes. + * main process. A first draft was reverted because the + * kernel-validated sender PID was not available on the existing + * socketpair setup. That blocker is gone: nxt_socketpair_create() + * sets SO_PASSCRED on both ends (and libunit does the same on its + * own sockets), NXT_HAVE_MSGHDR_CMSGCRED platforms get an SCM_CREDS + * control message attached to every send by + * nxt_socket_msg_oob_init(), and both receive paths in + * nxt_port_socket.c fill msg->cmsg_pid. Individual handlers are + * being gated on nxt_recv_msg_cmsg_pid() one at a time -- + * nxt_main_process_created_handler(), nxt_main_start_process_handler(), + * nxt_port_process_ready_handler(), nxt_proto_process_created_handler() + * and the cert/script/socket/access-log handlers already are. * - * Reverted to the pre-audit dispatch path here and deferred the - * ACL to a follow-up that introduces SO_PASSCRED on the relevant - * socketpairs and updates the process-registration ordering. + * What is left is the table-driven part: a per-message-type sender + * ACL applied here, in the dispatcher, so that a handler which + * forgets its own gate is still covered. That still needs the + * process-registration ordering sorted out -- dispatch can run + * before the prototype has finished registering a child, so the ACL + * cannot simply require an already registered sender. Platforms + * with neither SCM_CREDENTIALS nor SCM_CREDS (macOS, NetBSD, + * OpenBSD, illumos) stay unauthenticated either way. * See https://github.com/andypost/unit/pull/14 review thread. */ @@ -469,14 +481,14 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* Unreferenced: main and prototype processes run a single engine. */ /* - * The lookup stays on msg->port_msg.pid. The kernel-validated sender - * PID cannot replace it here: the prototype installs this handler too, - * and there SCM_CREDENTIALS carries the namespace-local pid of a - * pid-isolated worker, while the runtime hash is keyed on the global - * one (see nxt_proto_process_created_handler(), which uses the two as - * distinct values, and nxt_process_created() refusing to register an - * isolated pid). Authenticating this handler needs to accept either - * pid of the process it resolves; see the sender-ACL TODO above. + * The lookup stays on msg->port_msg.pid: it is the key of the runtime + * hash. The kernel-validated sender PID cannot replace it, because the + * prototype installs this handler too, and there SCM_CREDENTIALS + * carries the namespace-local pid of a pid-isolated worker while the + * hash is keyed on the global one (see nxt_proto_process_created_handler(), + * which uses the two as distinct values, and nxt_process_create() + * refusing to register an isolated pid). It authenticates the resolved + * process instead, right below. */ process = nxt_runtime_process_find(rt, msg->port_msg.pid); if (nxt_slow_path(process == NULL)) { @@ -484,14 +496,55 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } +#if (NXT_USE_CMSG_PID) /* - * A repeated PROCESS_READY is not rejected: the message is not - * authenticated, so refusing it would let a forged one arriving first - * make itself permanent and lock the legitimate sender out of its own - * queue. It replaces the mapping instead, as it did before this check - * existed, and the previous mapping is released below rather than - * leaked. nxt_assert() is not used because it compiles out in release - * builds, leaving no diagnostic at all. + * Authenticate the sender before anything else is read or written: the + * port socket of the main or the prototype process is duplicated into + * every child (nxt_proc_keep_matrix[]), the wire pid in the message is + * chosen by the sender, and the handler below re-points the named + * process's port at the queue this message carries. Without this gate + * any worker can announce a sibling and hand the receiver a shared + * memory queue of its own making. + * + * The credential is compared against process->isolated_pid, and only + * against it. isolated_pid is by construction the pid of the process + * as the receiver sees it -- the fork() return value in + * nxt_process_create(), left alone when nxt_proto_process_created_handler() + * rewrites process->pid to the global pid of a pid-isolated worker. + * Accepting process->pid as well would be a bypass: inside a pid + * namespace another task whose namespace-local pid happens to equal the + * victim's global pid would pass, and a compromised worker can fork + * towards such a collision. + * + * A non-positive isolated_pid means the record was not created by a + * fork() in this process -- nxt_runtime_process_get() only fills + * ->pid -- so there is nothing to authenticate against and no + * legitimate PROCESS_READY to lose: a worker announces itself to its + * parent prototype, and main's own children are all forked by main. + * Reject rather than compare, so that a cmsg_pid of 0 or the -1 that + * the shared-memory queue path leaves in place (see + * nxt_port_queue_read_handler()) cannot match by accident. + */ + if (nxt_slow_path(process->isolated_pid <= 0 + || nxt_recv_msg_cmsg_pid(msg) != process->isolated_pid)) + { + nxt_alert(task, "process %PI sent PROCESS_READY claiming process %PI", + nxt_recv_msg_cmsg_pid(msg), msg->port_msg.pid); + nxt_port_recv_msg_close_fds(msg); + return; + } +#endif + + /* + * A repeated PROCESS_READY from the authenticated owner is not + * rejected: on ucred and cmsgcred platforms the gate above has already + * turned a forged repeat into a rejection, and a genuine one has to + * keep working -- the process may legitimately re-announce a new queue, + * and refusing it would strand the process on the mapping it has. It + * replaces the mapping instead, as it did before this check existed, + * and the previous mapping is released below rather than leaked. + * nxt_assert() is not used because it compiles out in release builds, + * leaving no diagnostic at all. */ if (nxt_slow_path(process->state == NXT_PROCESS_STATE_READY)) { nxt_log(task, NXT_LOG_WARN, "repeated PROCESS_READY claiming " @@ -499,8 +552,8 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } /* - * Both guards reject before the state is set: a message that is not - * acted upon must leave no trace. Marking a process ready and only + * Every guard here rejects before the state is set: a message that is + * not acted upon must leave no trace. Marking a process ready and only * then rejecting it would make the next, legitimate PROCESS_READY trip * the guard above, and would let nxt_main_process_sigchld_handler() * clear the start stream of a process that never finished starting, @@ -565,14 +618,13 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) * unreachable copy; a QUIT can never arrive, while requests * still flow through the shared app queue. See issue #231. * - * The message is still not refused outright: it is not - * authenticated, and refusing it would let a forged - * PROCESS_READY carrying a bad descriptor keep the process - * from ever being marked ready. Failing the start is the - * honest reaction to a genuine mmap failure, but it hands the - * same forgery a way to kill a legitimate start, so it has to - * wait for sender authentication (see the lookup comment - * above). + * The message is still not refused outright. Failing the + * start is the honest reaction to a genuine mmap failure, and + * the sender gate above now makes that safe to do on ucred and + * cmsgcred platforms -- a forged PROCESS_READY can no longer + * use it to kill a legitimate start. The behaviour change + * itself, and the kill of the worker that is left with no way + * of being reached, is issue #231. */ if (port->queue == NULL) { nxt_alert(task, "process %PI ready: cannot map the queue; " diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index f60fe3f05..c92d081af 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -18,6 +18,13 @@ * * A message can carry two descriptors regardless of how many its type * normally uses, so the test attaches both and expects both back. + * + * The handler also authenticates the sender against the isolated_pid of + * the process the message names, wherever the platform hands the receiver + * a kernel-validated pid (NXT_USE_CMSG_PID). The fixture models a + * pid-isolated process -- pid and isolated_pid deliberately differ -- so + * that the cases below can tell the two apart: a credential equal to the + * global pid has to be refused just like a foreign one. */ #include @@ -96,6 +103,112 @@ nxt_port_ready_test_case(nxt_thread_t *thr, nxt_task_t *task, } +#if (NXT_USE_CMSG_PID) + +/* + * A PROCESS_READY whose kernel-validated sender pid does not match the + * isolated_pid of the process it names has to be refused outright: the + * descriptors it carries are closed, the process keeps the state it had, + * and the port keeps the queue it had. + * + * The message carries a mappable queue descriptor wherever one can be + * made, so that a handler without the gate is caught installing it rather + * than merely caught failing to map it -- which is what a /dev/null + * descriptor would produce, and would pass these checks on a queue that is + * already installed. + */ +static nxt_int_t +nxt_port_ready_test_reject(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_t *process, nxt_port_t *port, + nxt_pid_t cmsg_pid, const char *name) +{ + void *prev_queue; + nxt_fd_t fd0, fd1, prev_fd; + nxt_pid_t saved_cmsg_pid; + nxt_process_state_t prev_state; + +#if (NXT_HAVE_MEMFD_CREATE) + fd0 = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + + if (fd0 != -1 && ftruncate(fd0, sizeof(nxt_port_queue_t)) == -1) { + (void) close(fd0); + fd0 = -1; + } +#else + fd0 = open("/dev/null", O_RDONLY); +#endif + + fd1 = open("/dev/null", O_RDONLY); + + if (fd0 == -1 || fd1 == -1) { + nxt_log_alert(thr->log, "port ready test: %s failed to open the " + "descriptors", name); + goto fail; + } + + prev_state = process->state; + prev_queue = port->queue; + prev_fd = port->queue_fd; + + saved_cmsg_pid = msg->cmsg_pid; + msg->cmsg_pid = cmsg_pid; + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_port_process_ready_handler(task, msg); + + msg->cmsg_pid = saved_cmsg_pid; + + /* + * The state and the queue are checked before the descriptors: a + * handler that accepted the message owns fd0 through the port, and + * the cleanup below would close it a second time. + */ + if (process->state != prev_state) { + nxt_log_alert(thr->log, "port ready test: %s changed the process " + "state", name); + return NXT_ERROR; + } + + if (port->queue != prev_queue || port->queue_fd != prev_fd) { + nxt_log_alert(thr->log, "port ready test: %s replaced the queue", + name); + return NXT_ERROR; + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1) { + nxt_log_alert(thr->log, "port ready test: %s left fds in the message", + name); + return NXT_ERROR; + } + + if (nxt_port_ready_test_fd_is_open(fd0) + || nxt_port_ready_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "port ready test: %s leaked a descriptor", + name); + goto fail; /* closes only what is still open */ + } + + return NXT_OK; + +fail: + + if (fd0 != -1 && nxt_port_ready_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_port_ready_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + return NXT_ERROR; +} + +#endif + + #if (NXT_HAVE_MEMFD_CREATE) /* @@ -333,6 +446,7 @@ nxt_int_t nxt_port_ready_test(nxt_thread_t *thr) { nxt_mp_t *mp; + nxt_pid_t isolated_pid; nxt_task_t *task; nxt_int_t ret; nxt_port_t *port; @@ -363,6 +477,8 @@ nxt_port_ready_test(nxt_thread_t *thr) /* Defined before the first goto done: the cleanup there releases it. */ port = NULL; + isolated_pid = nxt_pid + 3; + mp = nxt_mp_create(1024, 128, 256, 32); if (nxt_slow_path(mp == NULL)) { return NXT_ERROR; @@ -410,13 +526,25 @@ nxt_port_ready_test(nxt_thread_t *thr) goto done; } + /* + * A pid-isolated process: pid is the global pid the runtime hash is + * keyed on, isolated_pid is the namespace-local pid the receiver reads + * out of SCM_CREDENTIALS. They differ on purpose -- with one value + * every case would pass the sender gate whichever of the two it + * compared, and the collision the gate exists to refuse could not be + * expressed at all. + */ process->pid = nxt_pid + 2; + process->isolated_pid = isolated_pid; process->state = NXT_PROCESS_STATE_CREATING; nxt_queue_init(&process->ports); nxt_runtime_process_add(task, process); msg.port_msg.pid = process->pid; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = process->isolated_pid; +#endif ret = nxt_port_ready_test_case(thr, task, &msg, "ready with no ports"); if (nxt_slow_path(ret != NXT_OK)) { @@ -434,13 +562,10 @@ nxt_port_ready_test(nxt_thread_t *thr) goto done; } -#if (NXT_HAVE_MEMFD_CREATE) - /* - * With a port on the process the handler takes the descriptor over and - * maps it, and a second PROCESS_READY replaces that mapping instead of - * leaking it -- a repeated message is not refused, so that a forged one - * arriving first cannot lock the legitimate sender out. + * From here on the process has a port, so the empty-ports guard can no + * longer be what refuses a message, and a refusal has to come from the + * sender gate itself. */ port = nxt_port_new(task, 1, process->pid, NXT_PROCESS_APP); if (nxt_slow_path(port == NULL)) { @@ -454,6 +579,54 @@ nxt_port_ready_test(nxt_thread_t *thr) nxt_queue_insert_tail(&process->ports, &port->link); +#if (NXT_USE_CMSG_PID) + + /* An unrelated sender announcing somebody else's process. */ + ret = nxt_port_ready_test_reject(thr, task, &msg, process, port, + nxt_pid + 4, "a foreign sender"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * The global pid of the very process being announced. Inside a pid + * namespace it is not the pid the receiver sees for that process, and + * another task's namespace-local pid can be made to equal it, so + * accepting it as well would leave the gate bypassable. + */ + ret = nxt_port_ready_test_reject(thr, task, &msg, process, port, + process->pid, "the global pid"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A record the receiver never forked keeps isolated_pid 0 -- + * nxt_runtime_process_get() fills only ->pid. There is nothing to + * authenticate against, so the credential of 0 that a plain equality + * test would accept has to be refused as well. + */ + process->isolated_pid = 0; + + ret = nxt_port_ready_test_reject(thr, task, &msg, process, port, 0, + "a process record nothing forked"); + + process->isolated_pid = isolated_pid; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#endif + +#if (NXT_HAVE_MEMFD_CREATE) + + /* + * The handler takes the descriptor over and maps it, and a second + * PROCESS_READY from the same, authenticated sender replaces that + * mapping instead of leaking it: a process may re-announce a new queue, + * and refusing the repeat would strand it on the mapping it has. + */ ret = nxt_port_ready_test_queue(thr, task, &msg, port, "first ready"); if (nxt_slow_path(ret != NXT_OK)) { goto done; @@ -465,6 +638,25 @@ nxt_port_ready_test(nxt_thread_t *thr) } ret = nxt_port_ready_test_short_queue(thr, task, &msg, port); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#if (NXT_USE_CMSG_PID) + + /* + * The same forgery once a queue is installed and the process is ready: + * this is the one that matters, because it is where an accepted message + * would re-point a live port at shared memory of the sender's choosing. + */ + ret = nxt_port_ready_test_reject(thr, task, &msg, process, port, + nxt_pid + 4, "a foreign sender re-arming " + "an installed queue"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#endif #endif From 2906191e98f522db818b486de7914cadc3c4ff9d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 17:10:52 +0200 Subject: [PATCH 069/391] router: clear proto_port_requests when a prototype start fails A prototype START_PROCESS that resolves through nxt_router_app_port_error() instead of nxt_router_app_port_ready() left app->proto_port_requests set, and nothing else ever cleared it. Only the success path reset the counter (nxt_router.c:4976-4977, which also replays that many parked starts). The error path never referenced it and did not even branch on app_joint_rpc->proto. Once the counter was stranded, every later nxt_router_start_app_process_handler() call for that application took the "wait for prototype process" branch at :430 and parked without sending anything -- so nothing remained that could produce another port_ready or port_error to clear it. The application could then never start another process for the router's lifetime. Recovery needs the nxt_app_t itself replaced: a fresh one is allocated only when the application's own config text changes (:1910-1920); a reload that leaves the block byte-identical reuses the wedged struct (:1935-1949). "restart application" bumps app->generation and QUITs an existing proto_port, but never touches proto_port_requests, and a wedged app has no prototype to QUIT in the first place. The stranded counter also stranded start slots. Every caller parked in the wait branch holds one app->pending_processes increment, taken by its initiator before the work was posted (nxt_router_start_app_process()), and the error path released exactly one -- so a cohort of N parked callers leaked N-1 slots on top of the wedge. That is the same counter #214 leaked, and it gates every future start. It also truncated the failure: nxt_router_app_start_failed() pops one waiter from app->ack_waiting_req, then keeps draining only while pending_processes is 0, so with a cohort of N>1 the counter never reached 0 and every waiter but the first sat until it timed out. So branch on app_joint_rpc->proto in nxt_router_app_port_error(): read and zero proto_port_requests under app->mutex, and release that many pending_processes slots. This mirrors the success path, which takes the same count and replays that many starts; the error path fails them instead. nxt_router_app_start_failed() grows a count parameter for it, and the ordinary callers pass 1, which is what they owned before. It also clamps count to the counter: the accounting cannot exceed it, but nxt_assert() is nothing in a release build, and over-releasing would wrap pending_processes to UINT32_MAX and wedge the application harder than the bug being fixed, while under-releasing merely leaks a slot. Distinct from #214/#236, which fixed the pre-arm leak at the "goto failed" label: those three sites never arm an RPC, so they leave proto_port_requests untouched. PR #236 listed this as deliberately out of its scope. All five accesses to the counter are on one thread, so the two increments need no mutex: nxt_port_post() runs the handler on port->engine's thread (nxt_port.c:932-941), the RPC is registered on that same port, and the read path asserts it dispatches there (nxt_port_socket.c:779, :887). The mutex taken here is for pending_processes, which other engines do read. src/test/nxt_router_proto_wedge_test.c drives the real arrangement -- no prototype port, so the handler builds a START_PROCESS payload and arms an RPC with ->proto set, then a second call joins the wait -- and tears the port's registrations down with nxt_port_rpc_close(), which is how a start that never answers is reported. Against the unfixed router it reports "proto_port_requests 2 (expected 0)" and the suite exits 1; with the fix the counter is clear, both parked slots are returned, and a later call sends a fresh prototype request instead of parking. The whole C suite is green in a normal and a --debug build. The fixture drains the main port on teardown rather than leaving the queued requests behind: nxt_port_msg_chk_insert() takes a port reference per queued message, and a port left above zero never reaches nxt_port_mp_cleanup(), whose assertions are the ones a --debug build is there to run. Confirmed under gdb that all three ports now reach it with use_count 0. Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + src/nxt_router.c | 70 ++++- src/test/nxt_router_proto_wedge_test.c | 346 +++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 5 files changed, 412 insertions(+), 10 deletions(-) create mode 100644 src/test/nxt_router_proto_wedge_test.c diff --git a/auto/sources b/auto/sources index 6d648eda6..d2d4cff8f 100644 --- a/auto/sources +++ b/auto/sources @@ -181,6 +181,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_new_port_test.c \ src/test/nxt_router_start_fail_test.c \ src/test/nxt_router_start_fail_soak_test.c \ + src/test/nxt_router_proto_wedge_test.c \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_fd_test.c \ " diff --git a/src/nxt_router.c b/src/nxt_router.c index 24106a090..913a137fa 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -229,7 +229,8 @@ static void nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static void nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); -static void nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app); +static void nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, + uint32_t count); static void nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i); static void nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app); @@ -503,7 +504,7 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, nxt_mp_free(b->data, b); } - nxt_router_app_start_failed(task, app); + nxt_router_app_start_failed(task, app, 1); skip: @@ -5054,6 +5055,7 @@ static void nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { + uint32_t n; nxt_app_t *app; nxt_app_joint_t *app_joint; nxt_app_joint_rpc_t *app_joint_rpc; @@ -5077,19 +5079,56 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_debug(task, "app '%V' %p start error", &app->name, app); - nxt_router_app_start_failed(task, app); + if (app_joint_rpc->proto) { + /* + * The prototype this RPC was armed for will never arrive, and only + * nxt_router_app_port_ready() clears proto_port_requests. Left set, + * it makes every later nxt_router_start_app_process_handler() call + * for this application take the "wait for prototype process" branch + * and park without sending anything -- so no further port_ready or + * port_error can ever fire to clear it. Only replacing the + * nxt_app_t recovers, and a reload does that only when the + * application's own config text changes. Clear it here, and + * release the slots of the whole parked cohort: the initiator's, + * plus one for each caller that joined the wait. This mirrors + * port_ready(), which takes the same count and replays that many + * starts on success. + */ + + nxt_thread_mutex_lock(&app->mutex); + + n = app->proto_port_requests; + app->proto_port_requests = 0; + + nxt_thread_mutex_unlock(&app->mutex); + + /* + * The counter is incremented in the same branch that sets ->proto, + * so an armed prototype RPC always owns at least its own slot. + */ + nxt_assert(n != 0); + + } else { + n = 1; + } + + nxt_router_app_start_failed(task, app, n); } /* - * A start attempt that will never yield a port: give back the - * pending_processes slot its initiator took and, if the application is left - * with no way to answer at all, fail the requests waiting for an - * acknowledgement instead of letting them sit until they time out. + * A start attempt that will never yield a port: give back the "count" + * pending_processes slots it owns and, if the application is left with no way + * to answer at all, fail the requests waiting for an acknowledgement instead + * of letting them sit until they time out. + * + * "count" is 1 for an ordinary attempt, which owns only its initiator's slot. + * A failed prototype attempt owns the whole parked cohort: see + * nxt_router_app_port_error(). */ static void -nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app) +nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, uint32_t count) { nxt_queue_link_t *link; nxt_http_request_t *r; @@ -5098,9 +5137,20 @@ nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app) nxt_thread_mutex_lock(&app->mutex); - nxt_assert(app->pending_processes != 0); + nxt_assert(app->pending_processes >= count); - app->pending_processes--; + /* + * The accounting above guarantees this, but nxt_assert() is nothing in a + * release build and the two directions are not equally bad: releasing one + * slot too few merely leaks it, while releasing one too many wraps the + * counter to UINT32_MAX and makes nxt_router_app_can_start() false + * forever -- a worse wedge than the one this path exists to clear. + */ + if (nxt_slow_path(count > app->pending_processes)) { + count = app->pending_processes; + } + + app->pending_processes -= count; if (app->processes == 0 && !nxt_queue_is_empty(&app->ack_waiting_req)) { link = nxt_queue_first(&app->ack_waiting_req); diff --git a/src/test/nxt_router_proto_wedge_test.c b/src/test/nxt_router_proto_wedge_test.c new file mode 100644 index 000000000..afdf624fc --- /dev/null +++ b/src/test/nxt_router_proto_wedge_test.c @@ -0,0 +1,346 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the prototype-start wedge: a prototype START_PROCESS + * RPC that resolves through nxt_router_app_port_error() instead of + * nxt_router_app_port_ready() used to leave app->proto_port_requests set. + * + * Only port_ready() cleared that counter, so once it was stranded every + * later nxt_router_start_app_process_handler() call for the application + * took the "wait for prototype process" branch at src/nxt_router.c:430 and + * parked without sending anything. Nothing was left to send a prototype + * request, so no port_ready or port_error could fire to clear it: the + * application could never start another process for the router's lifetime, + * and a reload that left its config block unchanged reused the wedged + * nxt_app_t rather than replacing it. + * + * The stranded counter also stranded pending_processes. Each parked caller + * holds one increment its initiator took, but port_error() released exactly + * one -- so a cohort of N parked callers leaked N-1 start slots on top of + * the wedge. + * + * The arrangement here is the one the router really uses: no prototype port + * yet, so the handler builds a START_PROCESS payload and arms an RPC with + * ->proto set; a second call then joins the wait. Tearing the port's + * registrations down with nxt_port_rpc_close() drives the router's own + * error handler for the armed RPC, which is how a start that never answers + * is reported. The test then requires what the wedge denied: the counter + * cleared, every slot in the cohort given back, and a later call actually + * sending a fresh prototype request instead of parking. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_proto_wedge_test_app; + + +nxt_int_t +nxt_router_proto_wedge_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *main_port, *shared_port; + nxt_runtime_t *rt, *saved_rt; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router proto wedge test started"); + + ret = NXT_ERROR; + router_port = NULL; + main_port = NULL; + shared_port = NULL; + joint = NULL; + app_mutex = 0; + app = &nxt_router_proto_wedge_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * The prototype branch allocates its payload from + * task->thread->engine->mem_pool. Only the members the reached code + * touches are set; see src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * A prototype start addresses the main process. No queue and + * write_ready unset, so nxt_port_socket_write2() takes the + * nxt_port_msg_chk_insert() path and the message is simply queued: + * the write succeeds and the RPC stays armed, which is what this test + * needs. + */ + + main_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + rt->port_by_type[NXT_PROCESS_MAIN] = main_port; + + /* Only its two descriptors are read, and both travel as -1. */ + + shared_port = nxt_port_new(task, 2, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(shared_port == NULL)) { + goto done; + } + + shared_port->pair[0] = -1; + shared_port->pair[1] = -1; + shared_port->socket.fd = -1; + shared_port->queue_fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "proto-wedge-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + app->shared_port = shared_port; + app->max_processes = 8; + app->max_pending_processes = 4; + + /* No prototype port yet: the next call starts one. */ + app->proto_port = NULL; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 1 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: prototype start left " + "proto_port_requests %uD joint %uD app use %d " + "(expected 1, 2 and 1)", + app->proto_port_requests, joint->use_count, + (int) app->use_count); + goto done; + } + + /* + * A second caller finds the request already in flight and joins the + * wait, adding its own initiator increment to both counters. + */ + + app->pending_processes = 2; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 2 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: parked caller left " + "proto_port_requests %uD joint %uD app use %d " + "(expected 2, 2 and 1)", + app->proto_port_requests, joint->use_count, + (int) app->use_count); + goto done; + } + + /* + * The prototype never answers. Closing the port's registrations drives + * nxt_router_app_port_error() for the armed RPC -- the wedge is what it + * used to leave behind. + */ + + nxt_port_rpc_close(task, router_port); + + if (app->proto_port_requests != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: failed prototype left " + "proto_port_requests %uD (expected 0)", + app->proto_port_requests); + goto done; + } + + if (app->pending_processes != 0 || joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: failed prototype left " + "pending %uD joint %uD (expected 0 and 1)", + app->pending_processes, joint->use_count); + goto done; + } + + /* + * The behaviour the wedge denied: with no prototype port and the + * counter clear, a later call must send a fresh prototype request -- + * arming an RPC and taking a joint reference -- rather than parking in + * the wait branch, which takes neither. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 1 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: app wedged after a failed " + "prototype, proto_port_requests %uD joint %uD " + "app use %d (expected 1, 2 and 1)", + app->proto_port_requests, joint->use_count, + (int) app->use_count); + goto done; + } + + /* Release the second armed RPC the same way. */ + + nxt_port_rpc_close(task, router_port); + + if (app->pending_processes != 0 || joint->use_count != 1 + || app->proto_port_requests != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: second release left pending " + "%uD joint %uD proto_port_requests %uD " + "(expected 0, 1 and 0)", + app->pending_processes, joint->use_count, + app->proto_port_requests); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router proto wedge test passed"); + + ret = NXT_OK; + +done: + + if (shared_port != NULL) { + nxt_port_use(task, shared_port, -1); + } + + if (main_port != NULL) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + + /* + * Both prototype requests were queued rather than written, and + * nxt_port_msg_chk_insert() takes a port reference for each. Left + * alone the port never reaches nxt_port_mp_cleanup(), so its pool + * and messages leak and the assertions there never run -- which + * would quietly hollow out this test under a debug build. + */ + nxt_port_test_run_error_handler(task, main_port); + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto wedge test: main port still holds " + "queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_use(task, main_port, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 3065670c0..4653f1e61 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -209,6 +209,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_proto_wedge_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 660ef1f00..fa919f02f 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -76,6 +76,7 @@ nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); +nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); From 6c3578bea3179cd3035b0726efd76c0fc558dce9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 22:52:31 +0200 Subject: [PATCH 070/391] port: fail the start when a worker's queue cannot be mapped A PROCESS_READY carrying a queue descriptor that cannot be mapped left the worker marked ready and announced to every peer, on a port nothing could be delivered on. The broadcast forwards queue_fd == -1, so each peer holds the same unreachable copy and a QUIT can never arrive, while requests still flow through the shared app queue. Only libunit attaches a queue to PROCESS_READY, and a libunit process emits a socket message only after dequeuing the READ_SOCKET marker that nxt_port_socket_write2() writes under port->queue != NULL. A queueless sender cannot emit that marker, so messages to it are suspended undelivered and the second one fails the worker with "too many port socket messages" -- the symptom reported against log rotation in #212. The start slot is never given back, so the application wedges. This is issue #231. Fail the start instead. The port->queue == NULL arm becomes an early return that closes the descriptors the message carried, marks the process, kills the worker, and returns without marking it ready or announcing it. Only the first mapping is a failed start. A PROCESS_READY that cannot replace an already installed queue keeps the working one and stays ready, exactly as before: refusing a replacement strands nothing. The whole arm, not merely the kill, is confined to the platforms that authenticate the sender (NXT_USE_CMSG_PID -- ucred or cmsgcred; see nxt_port.h). Every step of it is destructive, and none of it is safe on a message that cannot be attributed. Two things go wrong without that confinement, and both are worse than the bug being fixed: - Nothing reaps the worker. The kill is what makes the prototype send the REMOVE_PID that fails the start, so without it the start is pending forever rather than failed -- and the latch below then refuses every retransmission. - Any worker could forge a PROCESS_READY naming a sibling, with an unmappable descriptor, and latch that sibling out of ever becoming ready. Before this change no comparable durable state existed. So on macOS, NetBSD, OpenBSD and illumos the old behaviour stands, alert and all, until PROCESS_READY and NEW_PORT carry provenance everywhere -- that is issue #223. process->state = READY and its debug move below the whole "if (msg->fd[0] != -1)" block. Moving them below the mmap alone would have been wrong: nxt_process_send_ready() writes PROCESS_READY to process->parent_port with fd -1 for every process started that way, so all of them skip the block entirely and must still reach the tail. The process is left at CREATED because nothing about it ever became ready, and on this path that costs the release nothing. Which process reports the death decides whether it could: an app worker announces itself to the prototype (nxt_application.c:1202-1205), and the prototype's SIGCHLD notifies the others for any state but CREATING (nxt_application.c:907), so CREATED and READY are equivalent there -- the stream REMOVE_PID carries was set by the prototype when it forked the worker (nxt_application.c:672) and is never zeroed. For a prototype or a core process it is main that reports, and there the READY-only zeroing at nxt_main_process.c:1108-1110 does apply; CREATED is the correct state for both. The kill uses isolated_pid: the pid as this receiver sees it, which inside a pid namespace differs from the global one. nxt_process.c uses the same identity rule when it kills a child whose cgroup setup failed. It is also required to be positive, which is redundant -- the sender gate rejects a non-positive isolated_pid before this arm is reachable, under the same #if -- and kept only because what it guards is kill(0, SIGKILL) on the caller's own process group. SIGKILL rather than SIGTERM, because the worker has served nothing and a compromised one can ignore SIGTERM, leaving the start pending with no escalation. A port QUIT cannot work -- that the port is undeliverable is the premise. A record still at CREATING is advanced to CREATED before the kill, because the kill only fails the start if somebody notices the death: the prototype's SIGCHLD handler notifies the others for any state but CREATING (nxt_application.c:907). The record is normally CREATED by then, and not by a race that could go either way -- every app worker sends PROCESS_CREATED (nxt_process.c:918) over the same socketpair it later sends READY on, so READY cannot overtake it. The window is a LOST message rather than a reordered one: if that write hits EAGAIN it is buffered on the port (nxt_port_socket.c:331-359) and the worker then enters init->start and never returns to its loop to flush it, while libunit makes the descriptor blocking (nxt_unit.c:633) and sends READY regardless. This is traced statically and has not been observed. It cannot arise for a pid-isolated worker at all: that record is not in the hash until the CREATED handler adds it (nxt_application.c:783-789), so this arm early-returns at the lookup. CREATED rather than READY: the worker never became usable, and on the paths main reports rather than the prototype, READY is what makes main clear the start stream (nxt_main_process.c:1108-1110). A new start_failed latch on nxt_process_t stops a retransmitted PROCESS_READY from signalling a second time: by then the pid has been reaped and may name an unrelated process. Later starts are request-driven, the same as for a worker that crashes during startup, so no backoff is added and no autonomous restart loop exists. Site 2, nxt_main_new_port_handler(), is deliberately untouched. By the time NEW_PORT reaches main the prototype has already marked the worker ready and broadcast it, and main already owns the port from WHOAMI, so dropping it would trade a wedge for silently missing CHANGE_FILE and QUIT. NEW_PORT also has no provenance check. That is issue #223. Tests. nxt_port_ready_test_queueless() drives the first-mapping arm with a fresh process and a port that has no queue -- the existing short-queue case cannot reach it, because it runs after two successful installs and takes the queue != NULL arm, which is also the case pinned here as one that must not kill. The kill target is a real forked child rather than nxt_pid + N, since the suite runs as root; it is reaped with WNOHANG and a bounded retry, so a handler that never signals fails the suite instead of hanging it, and the status is required to be SIGKILL. The unauthenticated arm is asserted too: a change that quietly enabled the destructive path where the sender cannot be checked fails that case. The latch is tested for what it is for. A second live child is forked, isolated_pid and the sender credential are pointed at it, and a retransmitted PROCESS_READY must leave it running -- polled for 200ms, because SIGKILL is asynchronous and a single WNOHANG would race a handler that did signal and read as "survived". Confirmed by mutation: with the latch check disabled the child is reaped and the test fails, and without the polling window that same mutation passes. An NXT_TESTS counter on nxt_port_send_new_port() answers "was it announced". The function is nxt_inline and skips both the announced process and the receiver, so a fixture cannot distinguish "not broadcast" from "broadcast to nobody" by watching peers. The refused case must not broadcast, and the kept-queue case must -- without that positive half the assertion would pass on a handler that never broadcasts at all. nxt_router_remove_pid_soak_test.c runs the accounting 1000 times through the real route: it arms a start, reads the stream out of the START_PROCESS message the start actually queued, delivers a last-marked REMOVE_PID through the router's published handler table, and requires pending_processes back at its baseline. Baselines alternate because an over-release at baseline 0 is the case that hides -- since #255 the count is clamped to pending_processes rather than wrapping, so the counter simply stays at 0 -- and a stream nothing registered is delivered first, so the release cannot be credited to REMOVE_PID's other bookkeeping. Both new fixtures drain their destination port before releasing it. A queued message holds a port reference, and a port left above zero never reaches nxt_port_mp_cleanup(), whose assertions are what a --debug build is for (the pattern from #262, still open). Verified: C suite green in normal and --debug builds. Negative controls -- with the early return removed the suite exits 1 on "a refused queue was broadcast anyway", against master's handler on "a refused queue still marked the process ready", with the latch ignored on "a retransmitted READY killed again", and with the CREATING advance removed on "a refused queue left the process at CREATING, so nothing will notify its death". Compiled clean under -Werror with NXT_HAVE_UCRED unset and with NXT_HAVE_MEMFD_CREATE unset; on the former the emitted object carries only the unauthenticated wording, confirming the destructive arm is compiled out. PHP regression suite 62 passed, including both SIGUSR1 log-rotation cases and reconfiguration. Closes #231 Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + src/nxt_port.c | 142 ++++++- src/nxt_port.h | 8 + src/nxt_port_socket.c | 2 + src/nxt_process.h | 14 + src/test/nxt_port_ready_test.c | 361 +++++++++++++++++- src/test/nxt_router_remove_pid_soak_test.c | 423 +++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 9 files changed, 951 insertions(+), 5 deletions(-) create mode 100644 src/test/nxt_router_remove_pid_soak_test.c diff --git a/auto/sources b/auto/sources index d2d4cff8f..0f97d65e4 100644 --- a/auto/sources +++ b/auto/sources @@ -182,6 +182,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_start_fail_test.c \ src/test/nxt_router_start_fail_soak_test.c \ src/test/nxt_router_proto_wedge_test.c \ + src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_fd_test.c \ " diff --git a/src/nxt_port.c b/src/nxt_port.c index 7557f1996..ff1a9142e 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -318,6 +318,10 @@ nxt_port_send_new_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_port_t *port; nxt_process_t *process; +#if (NXT_TESTS) + nxt_port_test_broadcasts++; +#endif + nxt_debug(task, "new port %d for process %PI", new_port->pair[1], new_port->pid); @@ -566,12 +570,20 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - process->state = NXT_PROCESS_STATE_READY; + /* + * The start already failed and the process was killed for it. Acting on + * a retransmission would signal that pid a second time, and once it has + * been reaped the pid may name an unrelated process. + */ + if (nxt_slow_path(process->start_failed)) { + nxt_log(task, NXT_LOG_WARN, "PROCESS_READY claiming process %PI, " + "whose start has already failed", msg->port_msg.pid); + nxt_port_recv_msg_close_fds(msg); + return; + } port = nxt_process_port_first(process); - nxt_debug(task, "process %PI ready", msg->port_msg.pid); - if (msg->fd[0] != -1) { mem = nxt_port_queue_mmap(task, msg->fd[0], sizeof(nxt_port_queue_t)); @@ -627,9 +639,123 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) * of being reached, is issue #231. */ if (port->queue == NULL) { +#if (NXT_USE_CMSG_PID) + /* + * The whole arm, not only the kill, is confined to the + * platforms that authenticate the sender. Every step of it + * is destructive -- it refuses the announcement, strands the + * process at CREATED and signals it -- and none of that is + * safe to do on a message that cannot be attributed. + * + * Acting on an unauthenticated message would also be worse + * than the bug it fixes: without the kill below there is + * nothing to reap the worker, so the prototype never sends + * the REMOVE_PID that fails the start, and the latch refuses + * every retransmission -- a start pending forever rather + * than one that fails. Elsewhere the old behaviour stands + * until NEW_PORT and PROCESS_READY carry provenance + * everywhere; that is issue #223. + */ + + nxt_alert(task, "process %PI ready: cannot map the queue; " + "failing the start", msg->port_msg.pid); + + /* + * The tail below is not reached, so close what the sender + * attached here instead: the descriptor that could not be + * mapped, and fd[1], which this handler never uses. + */ + nxt_port_recv_msg_close_fds(msg); + + /* + * A retransmitted PROCESS_READY must not signal again: the + * pid has been killed and, once reaped, may name an + * unrelated process. The state cannot carry this, because + * the process has to stay at CREATED. + */ + process->start_failed = 1; + + /* + * The kill only fails the start if somebody notices the + * death, and the prototype's SIGCHLD handler notifies the + * others for any state but CREATING + * (nxt_application.c:907). + * + * The record is normally CREATED by then: the worker sends + * PROCESS_CREATED first (nxt_process.c:918) and the + * prototype sets the state when it handles it + * (nxt_application.c:769). Both messages travel the same + * socketpair, so READY cannot overtake CREATED. It can + * however be LOST: if that write hits EAGAIN the message is + * buffered on the port (nxt_port_socket.c:331-359) and the + * worker then enters init->start and never returns to its + * loop to flush it, while libunit makes the descriptor + * blocking and sends READY regardless. The record is then + * still CREATING here. + * + * Advancing it is what guarantees the REMOVE_PID that + * carries the start stream; leaving it at CREATING would + * kill the worker and still leave the start pending, which + * is the wedge this arm exists to end. + * + * CREATED, not READY: it never became usable, and on the + * paths main reports rather than the prototype, READY is + * what makes main clear the start stream + * (nxt_main_process.c:1108-1110). + */ + if (process->state == NXT_PROCESS_STATE_CREATING) { + process->state = NXT_PROCESS_STATE_CREATED; + } + + /* + * The process is left at CREATED because nothing about it + * ever became ready, and that costs the release nothing: + * an app worker announces itself to the prototype, whose + * SIGCHLD handler notifies the others whenever the state is + * anything but CREATING (nxt_application.c:907), so CREATED + * and READY travel the same path. The stream that REMOVE_PID + * carries -- the one the router turns into the start's + * RPC_ERROR -- was set by the prototype when it forked the + * worker (nxt_application.c:672) and is untouched here. + * + * isolated_pid is the pid as this receiver sees it, which + * inside a pid namespace is a different number from the + * global one; nxt_process.c uses the same identity rule + * when it kills a child whose cgroup setup failed. + * + * The positive test is redundant -- the sender gate above + * rejects a non-positive isolated_pid before this arm is + * reachable, under the same #if -- and is kept only because + * what it guards is kill(0, SIGKILL), which would signal + * the caller's whole process group. A later change that + * moved or relaxed that gate would otherwise turn this into + * exactly that. + */ + if (nxt_fast_path(process->isolated_pid > 0)) { + if (kill(process->isolated_pid, SIGKILL) == -1) { + /* + * ESRCH is ordinary here: the process may already + * have exited with a SIGCHLD still pending. + */ + nxt_log(task, (nxt_errno == ESRCH) ? NXT_LOG_INFO + : NXT_LOG_ALERT, + "kill(%PI, SIGKILL) failed for a process " + "whose queue could not be mapped %E", + process->isolated_pid, nxt_errno); + } + } + + return; +#else + /* + * See above: with no kernel-validated sender pid the start + * cannot be failed safely, so the announcement stands and + * the port stays unreachable. This is issue #231 as it was. + */ nxt_alert(task, "process %PI ready: cannot map the queue; " "the process cannot be reached on this port", msg->port_msg.pid); +#endif } else { nxt_log(task, NXT_LOG_WARN, "process %PI ready: cannot map " @@ -639,6 +765,16 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } } + /* + * Set below the mapping, not above it: a PROCESS_READY that cannot be + * acted upon must leave the process at CREATED (see the failure arm). + * Core processes -- router, controller, prototype -- send fd -1 and skip + * the block entirely, so they become READY here as they always did. + */ + process->state = NXT_PROCESS_STATE_READY; + + nxt_debug(task, "process %PI ready", msg->port_msg.pid); + /* * Close whatever the sender attached and the port did not take over: * fd[1] is never used here, and fd[0] survives a failed mapping. diff --git a/src/nxt_port.h b/src/nxt_port.h index 471350e75..e1aa65359 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -389,6 +389,14 @@ nxt_int_t nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, #if (NXT_TESTS) void nxt_port_test_msg_alloc_failures(nxt_uint_t failures); void nxt_port_test_run_error_handler(nxt_task_t *task, nxt_port_t *port); + +/* + * Counts entries into nxt_port_send_new_port(). It is nxt_inline and + * skips the announced process and the receiver itself, so a fixture with + * one process cannot tell "not broadcast" from "broadcast to nobody" by + * observing peers; counting the call itself distinguishes them. + */ +NXT_EXPORT extern nxt_uint_t nxt_port_test_broadcasts; #endif nxt_inline nxt_int_t diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 6677dd735..40bbc6860 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -40,6 +40,8 @@ static void nxt_port_error_handler(nxt_task_t *task, void *obj, void *data); #if (NXT_TESTS) static nxt_uint_t nxt_port_test_msg_alloc_failure_count; +nxt_uint_t nxt_port_test_broadcasts; + void nxt_port_test_msg_alloc_failures(nxt_uint_t failures) diff --git a/src/nxt_process.h b/src/nxt_process.h index 1c589d7b4..9575a09d1 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -123,6 +123,20 @@ struct nxt_process_s { */ nxt_bool_t released; + /* + * Latched when a PROCESS_READY could not be acted upon and the process + * was killed for it (nxt_port_process_ready_handler()). A retransmitted + * PROCESS_READY must not re-enter that arm and signal a pid that has + * already been killed -- and, once reaped, may name an unrelated + * process. + * + * The state cannot carry this, because the process is left at CREATED. + * Set only where the sender of a PROCESS_READY can be authenticated + * (NXT_USE_CMSG_PID); elsewhere that arm is not reachable, so nothing + * ever latches it. + */ + nxt_bool_t start_failed; + nxt_port_mmaps_t incoming; diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index c92d081af..a8e276a18 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -35,6 +35,8 @@ #include #include +#include +#include #if (NXT_HAVE_MEMFD_CREATE) #include @@ -411,8 +413,22 @@ nxt_port_ready_test_short_queue(nxt_thread_t *thr, nxt_task_t *task, msg->fd[0] = fd; msg->fd[1] = -1; + nxt_port_test_broadcasts = 0; + nxt_port_process_ready_handler(task, msg); + /* + * The positive half of the broadcast assertion in the queueless case: + * refusing a replacement is not a failed start, so this message still + * reaches the announcement. Without this, "not broadcast" there would + * pass on a handler that never broadcasts at all. + */ + if (nxt_port_test_broadcasts == 0) { + nxt_log_alert(thr->log, "port ready test: a kept queue was not " + "broadcast"); + return NXT_ERROR; + } + if (port->queue != prev_queue || port->queue_fd != prev_fd) { nxt_log_alert(thr->log, "port ready test: short queue replaced the " "installed one"); @@ -439,6 +455,294 @@ nxt_port_ready_test_short_queue(nxt_thread_t *thr, nxt_task_t *task, return NXT_OK; } +/* + * Issue #231: a first PROCESS_READY whose queue cannot be mapped. + * + * This is a different arm from nxt_port_ready_test_short_queue() above, + * and the distinction is the whole point. That case runs after a queue + * has been installed, so the handler keeps the working mapping and the + * process stays ready -- refusing a replacement is not a failed start. + * Here the port has no queue at all, so there is nothing to fall back to: + * the worker is unreachable on this port forever, and the start has to + * fail rather than be marked ready. + * + * The kill target is a real forked child, never nxt_pid + N. The suite + * runs as root in CI, so a made-up pid is a live process on the machine; + * only the pid fork() returned is known to be ours to signal. + */ +static nxt_int_t +nxt_port_ready_test_queueless(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_t *process, nxt_port_t *port) +{ + int status; + pid_t child, reaped; + nxt_fd_t fd; + nxt_uint_t i; + + if (nxt_slow_path(port->queue != NULL)) { + nxt_log_alert(thr->log, "port ready test: the queueless case needs a " + "port with no queue"); + return NXT_ERROR; + } + + /* + * A child that only waits to be killed. _exit() rather than return so + * that a child which somehow escapes pause() cannot run the rest of the + * test suite as a second process. + */ + child = fork(); + if (child == -1) { + nxt_log_alert(thr->log, "port ready test failed to fork the kill " + "target"); + return NXT_ERROR; + } + + if (child == 0) { + for ( ;; ) { + pause(); + } + + _exit(0); + } + + process->isolated_pid = child; + +#if (NXT_USE_CMSG_PID) + /* The sender gate compares against isolated_pid, which is now the child. */ + msg->cmsg_pid = child; +#endif + + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + if (fd == -1) { + nxt_log_alert(thr->log, "port ready test failed to create the queue"); + goto fail; + } + + /* One page: every byte past it faults, so the mapping must be refused. */ + if (ftruncate(fd, nxt_pagesize) == -1) { + nxt_log_alert(thr->log, "port ready test failed to size the queue"); + (void) close(fd); + goto fail; + } + + msg->fd[0] = fd; + msg->fd[1] = -1; + + nxt_port_test_broadcasts = 0; + + nxt_port_process_ready_handler(task, msg); + + if (msg->fd[0] != -1 || nxt_port_ready_test_fd_is_open(fd)) { + nxt_log_alert(thr->log, "port ready test: the refused queue leaked " + "its descriptor"); + goto fail; + } + + if (port->queue != NULL || port->queue_fd != -1) { + nxt_log_alert(thr->log, "port ready test: a refused queue was " + "installed anyway"); + goto fail; + } + +#if (NXT_USE_CMSG_PID) + + /* + * Never announce a port nothing can be delivered on: every peer would + * hold the same unreachable copy, and a QUIT could never arrive. The + * kept-queue case above is what stops this passing vacuously. + */ + if (nxt_port_test_broadcasts != 0) { + nxt_log_alert(thr->log, "port ready test: a refused queue was " + "broadcast anyway"); + goto fail; + } + + /* + * Not ready: nothing about this worker ever became usable, and the + * prototype's SIGCHLD notifies the others for any state but CREATING, + * so leaving it at CREATED still reaches the start's failure. + */ + if (process->state == NXT_PROCESS_STATE_READY) { + nxt_log_alert(thr->log, "port ready test: a refused queue still " + "marked the process ready"); + goto fail; + } + + if (!process->start_failed) { + nxt_log_alert(thr->log, "port ready test: a refused queue did not " + "mark the start as failed"); + goto fail; + } + + /* + * The fixture enters at CREATING, as a worker whose WHOAMI has not been + * handled yet would. Killing it while it is still CREATING would leave + * the prototype's SIGCHLD silent (nxt_application.c:907) and the start + * pending -- the wedge, reached by a different road. + */ + if (process->state != NXT_PROCESS_STATE_CREATED) { + nxt_log_alert(thr->log, "port ready test: a refused queue left the " + "process at CREATING, so nothing will notify its death"); + goto fail; + } + + /* + * Polled rather than blocking: a handler that never signalled would + * otherwise hang the suite instead of failing it. + */ + for (i = 0; i < 100; i++) { + reaped = waitpid(child, &status, WNOHANG); + + if (reaped == child) { + break; + } + + if (reaped == -1) { + nxt_log_alert(thr->log, "port ready test: waitpid() failed %E", + nxt_errno); + goto fail; + } + + nxt_nanosleep(10 * 1000000); /* 10ms, so at most a second. */ + } + + if (reaped != child) { + nxt_log_alert(thr->log, "port ready test: the process whose queue " + "was refused was not killed"); + goto fail; + } + + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) { + nxt_log_alert(thr->log, "port ready test: the killed process did not " + "die of SIGKILL (status %d)", status); + return NXT_ERROR; + } + + /* + * The latch, tested for what it is actually for. A retransmitted + * PROCESS_READY must not signal a second time: the first pid has been + * reaped and may by now name an unrelated process. Pointing + * isolated_pid at a second, live child makes that concrete -- without + * the latch the handler re-enters the failure arm and kills it, so the + * child's survival is the assertion. Re-checking the first pid would + * prove nothing, since it is already dead. + */ + child = fork(); + if (child == -1) { + nxt_log_alert(thr->log, "port ready test failed to fork the second " + "kill target"); + return NXT_ERROR; + } + + if (child == 0) { + for ( ;; ) { + pause(); + } + + _exit(0); + } + + process->isolated_pid = child; + msg->cmsg_pid = child; + + fd = syscall(SYS_memfd_create, "nxt_port_ready_test", MFD_CLOEXEC); + if (fd == -1 || ftruncate(fd, nxt_pagesize) == -1) { + nxt_log_alert(thr->log, "port ready test failed to re-create the " + "queue"); + if (fd != -1) { + (void) close(fd); + } + goto fail; + } + + msg->fd[0] = fd; + msg->fd[1] = -1; + + nxt_port_process_ready_handler(task, msg); + + if (msg->fd[0] != -1 || nxt_port_ready_test_fd_is_open(fd)) { + nxt_log_alert(thr->log, "port ready test: the retransmitted READY " + "leaked its descriptor"); + goto fail; + } + + if (process->state == NXT_PROCESS_STATE_READY) { + nxt_log_alert(thr->log, "port ready test: the retransmitted READY " + "marked the process ready"); + goto fail; + } + + /* + * Still running: the latch refused the arm before it reached the kill. + * + * SIGKILL is delivered asynchronously, so a single WNOHANG poll here + * would race a handler that did signal and read as "survived" -- the + * assertion has to give a kill time to land before concluding none + * happened. Verified by removing the latch check: without this window + * the mutation passes, with it the child is reaped and the test fails. + */ + for (i = 0; i < 20; i++) { + reaped = waitpid(child, &status, WNOHANG); + + if (reaped != 0) { + nxt_log_alert(thr->log, "port ready test: a retransmitted READY " + "killed again"); + return NXT_ERROR; + } + + nxt_nanosleep(10 * 1000000); /* 10ms, so 200ms in total. */ + } + + (void) kill(child, SIGKILL); + (void) waitpid(child, &status, 0); + +#else + + /* + * With no kernel-validated sender pid the start cannot be failed + * safely, so the handler keeps the pre-#231 behaviour and announces the + * process on a port that cannot be reached. Asserting that here is what + * keeps the guard honest: a change that quietly enabled the destructive + * arm on an unauthenticated platform fails this. + */ + (void) i; + (void) reaped; + + if (process->state != NXT_PROCESS_STATE_READY + || nxt_port_test_broadcasts == 0) + { + nxt_log_alert(thr->log, "port ready test: a platform with no sender " + "credential did not keep the announcing behaviour"); + goto fail; + } + + if (process->start_failed) { + nxt_log_alert(thr->log, "port ready test: a platform with no sender " + "credential latched a failed start"); + goto fail; + } + + /* Nothing signalled it, so the child is ours to clean up. */ + (void) kill(child, SIGKILL); + + if (waitpid(child, &status, 0) != child) { + nxt_log_alert(thr->log, "port ready test failed to reap the kill " + "target"); + return NXT_ERROR; + } + +#endif + + return NXT_OK; + +fail: + + (void) kill(child, SIGKILL); + (void) waitpid(child, &status, 0); + + return NXT_ERROR; +} + #endif @@ -449,9 +753,12 @@ nxt_port_ready_test(nxt_thread_t *thr) nxt_pid_t isolated_pid; nxt_task_t *task; nxt_int_t ret; - nxt_port_t *port; + nxt_port_t *port, *queueless_port; nxt_runtime_t *rt, *saved_rt; nxt_process_t *process; +#if (NXT_HAVE_MEMFD_CREATE) + nxt_process_t *queueless; +#endif nxt_port_recv_msg_t msg; nxt_thread_time_update(thr); @@ -474,8 +781,9 @@ nxt_port_ready_test(nxt_thread_t *thr) #endif - /* Defined before the first goto done: the cleanup there releases it. */ + /* Defined before the first goto done: the cleanup there releases them. */ port = NULL; + queueless_port = NULL; isolated_pid = nxt_pid + 3; @@ -642,6 +950,51 @@ nxt_port_ready_test(nxt_thread_t *thr) goto done; } + /* + * Issue #231 needs its own fixture: the process above is ready and its + * port has a working queue, which is exactly the case that must NOT + * fail a start. A fresh record, never announced, is the only way to + * reach the first-mapping arm. + */ + queueless = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(queueless == NULL)) { + ret = NXT_ERROR; + goto done; + } + + queueless->pid = nxt_pid + 5; + queueless->state = NXT_PROCESS_STATE_CREATING; + nxt_queue_init(&queueless->ports); + + nxt_runtime_process_add(task, queueless); + + queueless_port = nxt_port_new(task, 1, queueless->pid, NXT_PROCESS_APP); + if (nxt_slow_path(queueless_port == NULL)) { + ret = NXT_ERROR; + goto done; + } + + queueless_port->pair[0] = -1; + queueless_port->pair[1] = -1; + queueless_port->socket.fd = -1; + + nxt_queue_insert_tail(&queueless->ports, &queueless_port->link); + + msg.port_msg.pid = queueless->pid; + + ret = nxt_port_ready_test_queueless(thr, task, &msg, queueless, + queueless_port); + + /* Restore the message for anything that runs after this case. */ + msg.port_msg.pid = process->pid; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = process->isolated_pid; +#endif + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + #if (NXT_USE_CMSG_PID) /* @@ -671,6 +1024,10 @@ nxt_port_ready_test(nxt_thread_t *thr) nxt_port_close(task, port); } + if (queueless_port != NULL) { + nxt_port_close(task, queueless_port); + } + thr->runtime = saved_rt; nxt_thread_mutex_destroy(&rt->processes_mutex); diff --git a/src/test/nxt_router_remove_pid_soak_test.c b/src/test/nxt_router_remove_pid_soak_test.c new file mode 100644 index 000000000..72c288037 --- /dev/null +++ b/src/test/nxt_router_remove_pid_soak_test.c @@ -0,0 +1,423 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Accounting soak for the route a refused worker start really travels + * (issue #231). When a worker never becomes usable, nothing cancels its + * start RPC directly: the prototype reaps the child and sends REMOVE_PID + * carrying the stream the start was registered under, and the router turns + * that into the start failure. + * + * The chain this exercises, in the router's own order: + * + * nxt_router_remove_pid_handler() retypes a last-marked REMOVE_PID that + * carries a non-zero stream into + * _NXT_PORT_MSG_RPC_ERROR + * -> nxt_port_rpc_handler() finds and deletes the registration + * -> nxt_router_app_port_error() the RPC is a worker start, so ->proto + * is 0 and the count is 1 + * -> nxt_router_app_start_failed() gives that one slot back + * + * The handler is private to src/nxt_router.c; it is reached the way the + * port dispatcher reaches it, through the router's published handler table + * in nxt_router_process. + * + * Every iteration arms a real start through + * nxt_router_start_app_process_handler(), takes the stream from the + * START_PROCESS message that start actually queued for the prototype -- + * the very stream a prototype would echo back -- and requires + * pending_processes at its baseline afterwards. A leak leaves baseline+1 + * and a double release leaves baseline-1; both are caught without + * nxt_assert, so the check survives a non-debug build. + * + * The baselines alternate between 0 and a non-zero value because an + * over-release at baseline 0 is the case that hides: nxt_router.c clamps + * the count to pending_processes rather than letting it wrap, so the + * counter simply stays at 0 and an equality check against 0 cannot tell + * the difference. At a non-zero baseline the same bug shows up plainly. + * + * The counter app->proto_port_requests is required to stay 0 throughout. + * A worker start cannot touch it -- ->proto is set from the same condition + * that increments it, and this route never arms a prototype -- so the check + * is a guard against the fixture drifting into the prototype path, not a + * failure this route can produce. + * + * A stream nothing registered is delivered too, so the release above + * cannot be credited to anything other than the RPC registration. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_ROUTER_REMOVE_PID_SOAK_N 1000 + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_remove_pid_soak_test_app; + + +/* + * The stream a start put on the wire. nxt_port_socket_write2() queues the + * START_PROCESS message at the tail of the destination port, so the newest + * one is this iteration's; returns 0, which the caller rejects, if the + * write did not queue anything. + */ + +static uint32_t +nxt_router_remove_pid_soak_stream(nxt_port_t *dport) +{ + nxt_queue_link_t *link; + nxt_port_send_msg_t *msg; + + if (nxt_slow_path(nxt_queue_is_empty(&dport->messages))) { + return 0; + } + + link = nxt_queue_last(&dport->messages); + msg = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + + return msg->port_msg.stream; +} + + +/* + * A REMOVE_PID as the prototype sends it: the reaped worker's pid as the + * whole payload, the start stream, and last set so the registration is + * deleted rather than merely looked up. Modelled on the message + * nxt_port_rpc_remove_peer() synthesises for the same purpose. + */ + +static void +nxt_router_remove_pid_soak_msg(nxt_port_recv_msg_t *msg, nxt_buf_t *buf, + nxt_port_t *port, nxt_pid_t *pid, uint32_t stream) +{ + nxt_memzero(msg, sizeof(nxt_port_recv_msg_t)); + nxt_memzero(buf, sizeof(nxt_buf_t)); + + buf->mem.pos = (u_char *) pid; + buf->mem.free = (u_char *) (pid + 1); + + msg->fd[0] = -1; + msg->fd[1] = -1; + msg->buf = buf; + msg->port = port; + + msg->port_msg.stream = stream; + msg->port_msg.pid = nxt_pid; + msg->port_msg.type = _NXT_PORT_MSG_REMOVE_PID; + msg->port_msg.last = 1; +} + + +nxt_int_t +nxt_router_remove_pid_soak_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_buf_t buf; + nxt_pid_t worker_pid; + nxt_int_t ret; + uint32_t baseline, stream; + nxt_uint_t i; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *dport; + nxt_router_t router, *saved_router; + nxt_runtime_t *rt, *saved_rt; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test started"); + + ret = NXT_ERROR; + router_port = NULL; + dport = NULL; + joint = NULL; + app_mutex = 0; + app = &nxt_router_remove_pid_soak_test_app; + + /* + * No process by this pid is registered, so nxt_port_remove_pid() finds + * nothing to tear down and the message carries the stream alone -- the + * part of REMOVE_PID this test is about. + */ + worker_pid = nxt_pid + 1; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Only the members the reached code touches are set; see + * src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + /* + * nxt_router_remove_pid_handler() forwards the removal to every router + * engine before it reaches the RPC, and dereferences the nxt_router + * global to find them. An empty list is the honest arrangement here: + * this fixture runs no worker engines, and what the forwarding does -- + * dropping the dead process from each engine's app port hash -- is not + * part of the pending_processes accounting under test. + */ + nxt_memzero(&router, sizeof(nxt_router_t)); + nxt_queue_init(&router.engines); + + saved_router = nxt_router; + saved_rt = thr->runtime; + saved_engine = thr->engine; + nxt_router = &router; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on, and REMOVE_PID arrives on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The prototype the start addresses. No queue and write_ready unset, + * so nxt_port_socket_write2() takes the nxt_port_msg_chk_insert() path + * and the START_PROCESS message is simply queued: the write succeeds, + * the RPC stays armed, and the queued message is where the stream is + * read back from. + */ + + dport = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(dport == NULL)) { + goto done; + } + + dport->pair[0] = -1; + dport->pair[1] = -1; + dport->socket.fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "remove-pid-soak-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + app->max_processes = 8; + app->max_pending_processes = 4; + + /* A prototype port is present, so every start is a worker start. */ + app->proto_port = dport; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + /* Set once: a worker start must leave this alone. See the file head. */ + app->proto_port_requests = 0; + + for (i = 0; i < NXT_ROUTER_REMOVE_PID_SOAK_N; i++) { + baseline = (i % 2) ? 2 : 0; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = baseline + 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->pending_processes != baseline + 1 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui armed " + "start left pending %uD joint %uD app use %d " + "(expected %uD, 2 and 1)", + i, app->pending_processes, joint->use_count, + (int) app->use_count, baseline + 1); + goto done; + } + + stream = nxt_router_remove_pid_soak_stream(dport); + + if (nxt_slow_path(stream == 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui start " + "queued no START_PROCESS message to read a stream " + "from", i); + goto done; + } + + /* + * A stream no start registered: the handler must find no + * registration and change nothing. Without this the release below + * could be credited to any bookkeeping REMOVE_PID happens to do. + */ + + nxt_router_remove_pid_soak_msg(&msg, &buf, router_port, &worker_pid, + stream + 1); + + nxt_router_process.port_handlers->remove_pid(task, &msg); + + if (app->pending_processes != baseline + 1 || joint->use_count != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui " + "unregistered stream released pending %uD joint %uD " + "(expected %uD and 2)", + i, app->pending_processes, joint->use_count, + baseline + 1); + goto done; + } + + /* The real one: the worker died before it ever reported ready. */ + + nxt_router_remove_pid_soak_msg(&msg, &buf, router_port, &worker_pid, + stream); + + nxt_router_process.port_handlers->remove_pid(task, &msg); + + if (msg.port_msg.type != _NXT_PORT_MSG_RPC_ERROR + || msg.u.removed_pid != worker_pid) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui " + "REMOVE_PID was not turned into an RPC error for " + "pid %PI, type %d pid %PI", + i, worker_pid, (int) msg.port_msg.type, + msg.u.removed_pid); + goto done; + } + + if (app->pending_processes != baseline) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui " + "pending_processes %uD (expected %uD)", + i, app->pending_processes, baseline); + goto done; + } + + if (app->proto_port_requests != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui worker " + "start touched proto_port_requests %uD (expected 0)", + i, app->proto_port_requests); + goto done; + } + + if (joint->use_count != 1 || app->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: iteration %ui joint " + "use_count %uD app use_count %d (expected 1 and 1)", + i, joint->use_count, (int) app->use_count); + goto done; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test passed: %d starts failed " + "through REMOVE_PID", NXT_ROUTER_REMOVE_PID_SOAK_N); + + ret = NXT_OK; + +done: + + if (dport != NULL) { + app->proto_port = NULL; + + /* + * Every start queued its START_PROCESS message rather than writing + * it, and nxt_port_msg_chk_insert() takes a port reference for each. + * Left alone the port never reaches nxt_port_mp_cleanup(), so its + * pool and messages leak and the assertions there never run -- which + * would quietly hollow out this test under a debug build. + */ + nxt_port_test_run_error_handler(task, dport); + + if (!nxt_queue_is_empty(&dport->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router remove pid soak test: prototype port still " + "holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_use(task, dport, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + nxt_router = saved_router; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 4653f1e61..f14011224 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -213,6 +213,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_remove_pid_soak_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index fa919f02f..ba83f3160 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -77,6 +77,7 @@ nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); +nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); From 1596376a12f0cd37bf7c14bb0d2c0b176f6ee150 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 31 Aug 2026 01:20:39 +0200 Subject: [PATCH 071/391] port: fail the start when a READY carries no queue descriptor The previous commit failed the start when an application worker's queue descriptor could not be mapped. A descriptor that never arrives reaches the same wedge by a different road, and the check missed it. At the receiver's RLIMIT_NOFILE, Linux delivers a message's payload and its SCM_CREDENTIALS while discarding the SCM_RIGHTS descriptor and setting MSG_CTRUNC. Nothing in the receive path inspects that flag -- MSG_CTRUNC does not appear anywhere in src/ -- so the handler sees a PROCESS_READY that passes the sender gate with fd[0] == -1, skips the mapping block by design, and marks the worker ready. Its queueless port is then broadcast to every peer: the state issue #231 is about, arrived at without any mmap failure. For an application worker that can never be legitimate. Only libunit attaches a queue to PROCESS_READY and it always does (nxt_unit.c:1050, called with the descriptor the worker just created). The processes that do send fd -1 are the core ones -- nxt_process_send_ready() writes to process->parent_port, which is main -- and they are not NXT_PROCESS_APP. So test the outcome rather than the cause: after the mapping block, an NXT_PROCESS_APP port that still has no queue is a failed start, however the descriptor went missing. That replaces the previous inner test and is a simplification -- the two roads were always the same failure. A repeated READY whose replacement mapping fails is excluded by the same condition, because the port still holds the queue it had; refusing a replacement stays what it was, a warning. This does not fix MSG_CTRUNC. Every other handler that takes a descriptor -- NEW_PORT, mmap, CHANGE_FILE -- has the same blind spot, and surfacing truncation in nxt_recvmsg() as a receive error is a change to every message type with its own blast radius. That belongs in its own commit, and is filed separately; this one closes the door that issue #231 is about. The test injects fd[0] = -1 on an app record directly rather than exhausting descriptors: the kernel path is the motivation, and reproducing it would make the case depend on the receiver's rlimit rather than on the handler's decision. It uses its own fresh record, because the earlier case leaves its subject latched. Negative control: restoring the old "only when a descriptor was present" condition fails the suite on "a READY with no queue descriptor was broadcast anyway". Also fixes a defect in the previous commit's fixture that this case exposed: the new case's assignment to ret overwrote the earlier case's failure, so a queueless-case failure was reported as a pass. Checked before the second case runs now. The three earlier negative controls were re-run against the corrected fixture and all still fail the suite. Co-Authored-By: Claude Opus 5 --- src/nxt_port.c | 282 +++++++++++++++++---------------- src/test/nxt_port_ready_test.c | 170 +++++++++++++++++++- 2 files changed, 310 insertions(+), 142 deletions(-) diff --git a/src/nxt_port.c b/src/nxt_port.c index ff1a9142e..8dd9f1a61 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -613,156 +613,158 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* The port owns the descriptor now. */ msg->fd[0] = -1; - } else { - /* - * A refused queue leaves the port as it was. For a repeated - * PROCESS_READY that is the queue it already has, and the port - * keeps working. For the first one it is no queue at all, and - * that is not a fallback to the socket: only libunit attaches - * a queue descriptor to PROCESS_READY, and a libunit process - * delivers a socket message only after dequeuing the - * READ_SOCKET marker that nxt_port_socket_write2() emits under - * port->queue != NULL. A queueless sender cannot emit the - * marker, so its messages are suspended undelivered on the - * worker side, and the second one fails the worker's context - * ("too many port socket messages"). The broadcast below - * forwards queue_fd == -1, so every peer holds the same - * unreachable copy; a QUIT can never arrive, while requests - * still flow through the shared app queue. See issue #231. - * - * The message is still not refused outright. Failing the - * start is the honest reaction to a genuine mmap failure, and - * the sender gate above now makes that safe to do on ucred and - * cmsgcred platforms -- a forged PROCESS_READY can no longer - * use it to kill a legitimate start. The behaviour change - * itself, and the kill of the worker that is left with no way - * of being reached, is issue #231. - */ - if (port->queue == NULL) { + } else if (port->queue != NULL) { + nxt_log(task, NXT_LOG_WARN, "process %PI ready: cannot map " + "the replacement queue, keeping the current one", + msg->port_msg.pid); + } + } + + /* + * One test for both ways an application worker can end up with no queue: + * a descriptor that could not be mapped, and no descriptor at all. + * + * The second is not hypothetical. Only libunit attaches a queue to + * PROCESS_READY, and it always does (nxt_unit.c:1050) -- the processes + * that legitimately send fd -1 are the core ones, which announce to main + * and are not NXT_PROCESS_APP. But at the receiver's RLIMIT_NOFILE the + * kernel delivers the payload and the credential while discarding + * SCM_RIGHTS and setting MSG_CTRUNC, which nothing here inspects, so an + * authenticated READY can arrive with its descriptor silently gone. + * Treating that as ready would broadcast the same unreachable port that + * issue #231 is about, by a different road. + * + * A queueless APP port is therefore a failed start either way, and the + * distinction that matters is not how the descriptor was lost but that + * the worker cannot be reached on it: no fallback to the socket exists, + * because a libunit process only delivers a socket message after + * dequeuing the READ_SOCKET marker that nxt_port_socket_write2() emits + * under port->queue != NULL. A queueless sender never emits the marker, + * its messages are suspended undelivered, and the second one fails the + * worker with "too many port socket messages". + * + * A repeated READY whose replacement mapping failed keeps the queue it + * has and is not a failed start -- it is excluded here by port->queue. + */ + + if (port->queue == NULL && port->type == NXT_PROCESS_APP) { #if (NXT_USE_CMSG_PID) - /* - * The whole arm, not only the kill, is confined to the - * platforms that authenticate the sender. Every step of it - * is destructive -- it refuses the announcement, strands the - * process at CREATED and signals it -- and none of that is - * safe to do on a message that cannot be attributed. - * - * Acting on an unauthenticated message would also be worse - * than the bug it fixes: without the kill below there is - * nothing to reap the worker, so the prototype never sends - * the REMOVE_PID that fails the start, and the latch refuses - * every retransmission -- a start pending forever rather - * than one that fails. Elsewhere the old behaviour stands - * until NEW_PORT and PROCESS_READY carry provenance - * everywhere; that is issue #223. - */ + /* + * The whole arm, not only the kill, is confined to the + * platforms that authenticate the sender. Every step of it + * is destructive -- it refuses the announcement, strands the + * process at CREATED and signals it -- and none of that is + * safe to do on a message that cannot be attributed. + * + * Acting on an unauthenticated message would also be worse + * than the bug it fixes: without the kill below there is + * nothing to reap the worker, so the prototype never sends + * the REMOVE_PID that fails the start, and the latch refuses + * every retransmission -- a start pending forever rather + * than one that fails. Elsewhere the old behaviour stands + * until NEW_PORT and PROCESS_READY carry provenance + * everywhere; that is issue #223. + */ - nxt_alert(task, "process %PI ready: cannot map the queue; " - "failing the start", msg->port_msg.pid); + nxt_alert(task, "process %PI ready: cannot map the queue; " + "failing the start", msg->port_msg.pid); - /* - * The tail below is not reached, so close what the sender - * attached here instead: the descriptor that could not be - * mapped, and fd[1], which this handler never uses. - */ - nxt_port_recv_msg_close_fds(msg); + /* + * The tail below is not reached, so close what the sender + * attached here instead: the descriptor that could not be + * mapped, and fd[1], which this handler never uses. + */ + nxt_port_recv_msg_close_fds(msg); - /* - * A retransmitted PROCESS_READY must not signal again: the - * pid has been killed and, once reaped, may name an - * unrelated process. The state cannot carry this, because - * the process has to stay at CREATED. - */ - process->start_failed = 1; + /* + * A retransmitted PROCESS_READY must not signal again: the + * pid has been killed and, once reaped, may name an + * unrelated process. The state cannot carry this, because + * the process has to stay at CREATED. + */ + process->start_failed = 1; - /* - * The kill only fails the start if somebody notices the - * death, and the prototype's SIGCHLD handler notifies the - * others for any state but CREATING - * (nxt_application.c:907). - * - * The record is normally CREATED by then: the worker sends - * PROCESS_CREATED first (nxt_process.c:918) and the - * prototype sets the state when it handles it - * (nxt_application.c:769). Both messages travel the same - * socketpair, so READY cannot overtake CREATED. It can - * however be LOST: if that write hits EAGAIN the message is - * buffered on the port (nxt_port_socket.c:331-359) and the - * worker then enters init->start and never returns to its - * loop to flush it, while libunit makes the descriptor - * blocking and sends READY regardless. The record is then - * still CREATING here. - * - * Advancing it is what guarantees the REMOVE_PID that - * carries the start stream; leaving it at CREATING would - * kill the worker and still leave the start pending, which - * is the wedge this arm exists to end. - * - * CREATED, not READY: it never became usable, and on the - * paths main reports rather than the prototype, READY is - * what makes main clear the start stream - * (nxt_main_process.c:1108-1110). - */ - if (process->state == NXT_PROCESS_STATE_CREATING) { - process->state = NXT_PROCESS_STATE_CREATED; - } + /* + * The kill only fails the start if somebody notices the + * death, and the prototype's SIGCHLD handler notifies the + * others for any state but CREATING + * (nxt_application.c:907). + * + * The record is normally CREATED by then: the worker sends + * PROCESS_CREATED first (nxt_process.c:918) and the + * prototype sets the state when it handles it + * (nxt_application.c:769). Both messages travel the same + * socketpair, so READY cannot overtake CREATED. It can + * however be LOST: if that write hits EAGAIN the message is + * buffered on the port (nxt_port_socket.c:331-359) and the + * worker then enters init->start and never returns to its + * loop to flush it, while libunit makes the descriptor + * blocking and sends READY regardless. The record is then + * still CREATING here. + * + * Advancing it is what guarantees the REMOVE_PID that + * carries the start stream; leaving it at CREATING would + * kill the worker and still leave the start pending, which + * is the wedge this arm exists to end. + * + * CREATED, not READY: it never became usable, and on the + * paths main reports rather than the prototype, READY is + * what makes main clear the start stream + * (nxt_main_process.c:1108-1110). + */ + if (process->state == NXT_PROCESS_STATE_CREATING) { + process->state = NXT_PROCESS_STATE_CREATED; + } + /* + * The process is left at CREATED because nothing about it + * ever became ready, and that costs the release nothing: + * an app worker announces itself to the prototype, whose + * SIGCHLD handler notifies the others whenever the state is + * anything but CREATING (nxt_application.c:907), so CREATED + * and READY travel the same path. The stream that REMOVE_PID + * carries -- the one the router turns into the start's + * RPC_ERROR -- was set by the prototype when it forked the + * worker (nxt_application.c:672) and is untouched here. + * + * isolated_pid is the pid as this receiver sees it, which + * inside a pid namespace is a different number from the + * global one; nxt_process.c uses the same identity rule + * when it kills a child whose cgroup setup failed. + * + * The positive test is redundant -- the sender gate above + * rejects a non-positive isolated_pid before this arm is + * reachable, under the same #if -- and is kept only because + * what it guards is kill(0, SIGKILL), which would signal + * the caller's whole process group. A later change that + * moved or relaxed that gate would otherwise turn this into + * exactly that. + */ + if (nxt_fast_path(process->isolated_pid > 0)) { + if (kill(process->isolated_pid, SIGKILL) == -1) { /* - * The process is left at CREATED because nothing about it - * ever became ready, and that costs the release nothing: - * an app worker announces itself to the prototype, whose - * SIGCHLD handler notifies the others whenever the state is - * anything but CREATING (nxt_application.c:907), so CREATED - * and READY travel the same path. The stream that REMOVE_PID - * carries -- the one the router turns into the start's - * RPC_ERROR -- was set by the prototype when it forked the - * worker (nxt_application.c:672) and is untouched here. - * - * isolated_pid is the pid as this receiver sees it, which - * inside a pid namespace is a different number from the - * global one; nxt_process.c uses the same identity rule - * when it kills a child whose cgroup setup failed. - * - * The positive test is redundant -- the sender gate above - * rejects a non-positive isolated_pid before this arm is - * reachable, under the same #if -- and is kept only because - * what it guards is kill(0, SIGKILL), which would signal - * the caller's whole process group. A later change that - * moved or relaxed that gate would otherwise turn this into - * exactly that. - */ - if (nxt_fast_path(process->isolated_pid > 0)) { - if (kill(process->isolated_pid, SIGKILL) == -1) { - /* - * ESRCH is ordinary here: the process may already - * have exited with a SIGCHLD still pending. - */ - nxt_log(task, (nxt_errno == ESRCH) ? NXT_LOG_INFO - : NXT_LOG_ALERT, - "kill(%PI, SIGKILL) failed for a process " - "whose queue could not be mapped %E", - process->isolated_pid, nxt_errno); - } - } - - return; -#else - /* - * See above: with no kernel-validated sender pid the start - * cannot be failed safely, so the announcement stands and - * the port stays unreachable. This is issue #231 as it was. + * ESRCH is ordinary here: the process may already + * have exited with a SIGCHLD still pending. */ - nxt_alert(task, "process %PI ready: cannot map the queue; " - "the process cannot be reached on this port", - msg->port_msg.pid); -#endif - - } else { - nxt_log(task, NXT_LOG_WARN, "process %PI ready: cannot map " - "the replacement queue, keeping the current one", - msg->port_msg.pid); + nxt_log(task, (nxt_errno == ESRCH) ? NXT_LOG_INFO + : NXT_LOG_ALERT, + "kill(%PI, SIGKILL) failed for a process " + "whose queue could not be mapped %E", + process->isolated_pid, nxt_errno); } } + + return; +#else + /* + * See above: with no kernel-validated sender pid the start + * cannot be failed safely, so the announcement stands and + * the port stays unreachable. This is issue #231 as it was. + */ + nxt_alert(task, "process %PI ready: cannot map the queue; " + "the process cannot be reached on this port", + msg->port_msg.pid); +#endif } /* diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index a8e276a18..327041e77 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -455,6 +455,125 @@ nxt_port_ready_test_short_queue(nxt_thread_t *thr, nxt_task_t *task, return NXT_OK; } +/* + * The same wedge reached without any mmap failure: an authenticated + * PROCESS_READY for an application worker that carries no descriptor at all. + * + * At the receiver's RLIMIT_NOFILE the kernel delivers the payload and the + * SCM_CREDENTIALS while discarding SCM_RIGHTS and setting MSG_CTRUNC, which + * nothing in the receive path inspects -- so the handler sees a message that + * passes the sender gate with fd[0] == -1. Only libunit attaches a queue to + * PROCESS_READY and it always does, so for an NXT_PROCESS_APP port this can + * never be legitimate; the core processes that do send fd -1 announce to main + * and are not APP. + * + * The fd is injected directly rather than by exhausting descriptors: the + * kernel path is the motivation, and reproducing it would make the test + * depend on the receiver's rlimit rather than on the handler's decision. + */ +static nxt_int_t +nxt_port_ready_test_no_queue_fd(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_t *process, nxt_port_t *port) +{ + int status; + pid_t child, reaped; + + if (nxt_slow_path(port->queue != NULL + || port->type != NXT_PROCESS_APP)) + { + nxt_log_alert(thr->log, "port ready test: the no-queue-fd case needs " + "an app port with no queue"); + return NXT_ERROR; + } + + child = fork(); + if (child == -1) { + nxt_log_alert(thr->log, "port ready test failed to fork the kill " + "target"); + return NXT_ERROR; + } + + if (child == 0) { + for ( ;; ) { + pause(); + } + + _exit(0); + } + + process->isolated_pid = child; + process->start_failed = 0; + process->state = NXT_PROCESS_STATE_CREATING; + +#if (NXT_USE_CMSG_PID) + msg->cmsg_pid = child; +#endif + + /* What a truncated control block leaves behind. */ + msg->fd[0] = -1; + msg->fd[1] = -1; + + nxt_port_test_broadcasts = 0; + + nxt_port_process_ready_handler(task, msg); + +#if (NXT_USE_CMSG_PID) + + if (nxt_port_test_broadcasts != 0 || port->queue != NULL) { + nxt_log_alert(thr->log, "port ready test: a READY with no queue " + "descriptor was broadcast anyway"); + goto fail; + } + + if (process->state == NXT_PROCESS_STATE_READY || !process->start_failed) { + nxt_log_alert(thr->log, "port ready test: a READY with no queue " + "descriptor still marked the process ready"); + goto fail; + } + + if (waitpid(child, &status, 0) != child) { + nxt_log_alert(thr->log, "port ready test failed to reap the kill " + "target"); + return NXT_ERROR; + } + + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) { + nxt_log_alert(thr->log, "port ready test: the worker that announced " + "no queue did not die of SIGKILL (status %d)", status); + return NXT_ERROR; + } + +#else + + if (process->state != NXT_PROCESS_STATE_READY) { + nxt_log_alert(thr->log, "port ready test: a platform with no sender " + "credential did not keep the announcing behaviour"); + goto fail; + } + + (void) kill(child, SIGKILL); + + if (waitpid(child, &status, 0) != child) { + nxt_log_alert(thr->log, "port ready test failed to reap the kill " + "target"); + return NXT_ERROR; + } + +#endif + + (void) reaped; + + return NXT_OK; + +fail: + + (void) kill(child, SIGKILL); + (void) waitpid(child, &status, 0); + + return NXT_ERROR; +} + + /* * Issue #231: a first PROCESS_READY whose queue cannot be mapped. * @@ -753,11 +872,11 @@ nxt_port_ready_test(nxt_thread_t *thr) nxt_pid_t isolated_pid; nxt_task_t *task; nxt_int_t ret; - nxt_port_t *port, *queueless_port; + nxt_port_t *port, *queueless_port, *nofd_port; nxt_runtime_t *rt, *saved_rt; nxt_process_t *process; #if (NXT_HAVE_MEMFD_CREATE) - nxt_process_t *queueless; + nxt_process_t *queueless, *nofd; #endif nxt_port_recv_msg_t msg; @@ -784,6 +903,7 @@ nxt_port_ready_test(nxt_thread_t *thr) /* Defined before the first goto done: the cleanup there releases them. */ port = NULL; queueless_port = NULL; + nofd_port = NULL; isolated_pid = nxt_pid + 3; @@ -985,6 +1105,48 @@ nxt_port_ready_test(nxt_thread_t *thr) ret = nxt_port_ready_test_queueless(thr, task, &msg, queueless, queueless_port); + /* + * Checked before the next case runs: its own assignment to ret would + * otherwise overwrite this failure and the suite would report a pass. + */ + if (nxt_slow_path(ret != NXT_OK)) { + msg.port_msg.pid = process->pid; + goto done; + } + + /* + * A second fresh record for the truncated-control case: the one above + * has its start_failed latch set, which would refuse the message before + * the arm under test is reached. + */ + nofd = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(nofd == NULL)) { + ret = NXT_ERROR; + goto done; + } + + nofd->pid = nxt_pid + 6; + nofd->state = NXT_PROCESS_STATE_CREATING; + nxt_queue_init(&nofd->ports); + + nxt_runtime_process_add(task, nofd); + + nofd_port = nxt_port_new(task, 1, nofd->pid, NXT_PROCESS_APP); + if (nxt_slow_path(nofd_port == NULL)) { + ret = NXT_ERROR; + goto done; + } + + nofd_port->pair[0] = -1; + nofd_port->pair[1] = -1; + nofd_port->socket.fd = -1; + + nxt_queue_insert_tail(&nofd->ports, &nofd_port->link); + + msg.port_msg.pid = nofd->pid; + + ret = nxt_port_ready_test_no_queue_fd(thr, task, &msg, nofd, nofd_port); + /* Restore the message for anything that runs after this case. */ msg.port_msg.pid = process->pid; #if (NXT_USE_CMSG_PID) @@ -1028,6 +1190,10 @@ nxt_port_ready_test(nxt_thread_t *thr) nxt_port_close(task, queueless_port); } + if (nofd_port != NULL) { + nxt_port_close(task, nofd_port); + } + thr->runtime = saved_rt; nxt_thread_mutex_destroy(&rt->processes_mutex); From 9d40631ad6408105e3be697f10fa2839fab41aa2 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 18:03:03 +0200 Subject: [PATCH 072/391] test(php): provision a timezone database in the isolation rootfs test_php_isolation_rootfs was skipped on PHP >= 8.5 as "SIGSEGV under investigation". The version gate is wrong: the crash is not a PHP 8.5 defect and not a Unit defect. PHP built with --with-system-tzdata -- every Debian, Ubuntu and Fedora package -- opens the timezone database under /usr/share/zoneinfo lazily, at request time, which is after Unit has chroot()ed the app worker into "rootfs". The test hands PHP an empty rootfs, so the lookup returns NULL and the distro patch dereferences it: the worker dies on signal 11 and Unit answers 503. PHP 8.3.33, 8.4.24, 8.4.25 and 8.5.4 all crash the same way; the official php:*-cli images pass only because they carry timelib's bundled database and never touch the filesystem. Copy the host's timezone database into the rootfs alongside the app, and drop the version skip. Absolute symlinks are left out of the copy: the teardown in conftest chmods every file it walks in the temp dir, so localtime -> /etc/localtime would carry it out of the rootfs and leave the host's own zone file mode 0777. When the host has no /usr/share/zoneinfo at all the rootfs cannot be provisioned, so skip with that reason rather than fail. Also print the PHP timezone database in the CI PHP leg, so the runner's build flavour is visible in the log. Co-Authored-By: Claude Opus 5 --- .github/workflows/build-test.yml | 10 ++++++++ TODO.md | 19 --------------- test/test_php_isolation.py | 6 ----- test/unit/applications/lang/php.py | 37 ++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 3a41e92cf..2305b98d5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -348,6 +348,16 @@ jobs: update: true if: steps.metadata.outputs.module == 'php' + # Informational. A package built --with-system-tzdata reports + # "Version => 0.system" and reads /usr/share/zoneinfo at request time, + # i.e. inside an isolation.rootfs chroot; a bundled-timelib build + # reports a real release and never touches the filesystem. Which one + # the runner has decides how test_php_isolation_rootfs behaves. + - name: Report php timezone database + run: | + php -i | grep -i 'timezone database' || true + if: steps.metadata.outputs.module == 'php' + - name: Configure php run: | ./configure php diff --git a/TODO.md b/TODO.md index e6e7d6ceb..aa37926a9 100644 --- a/TODO.md +++ b/TODO.md @@ -48,7 +48,6 @@ **Open questions (decide before August):** - [ ] PHP TrueAsync — determine source of `nxt_php_extension.c` (fork EdmondDantes or write from scratch) -- [ ] PHP 8.5 `rootfs` SIGSEGV — needs diagnostics on real hardware - [ ] OpenSSL 3.6 migration — verify clang-ast compatibility - [ ] Proxy request buffering (#58) — define scope (per-action vs global) @@ -485,24 +484,6 @@ causing `test_php_application_disable_classes` and `test_php_application_disable --- -### `rootfs` isolation SIGSEGV (PHP 8.5) - -`test_php_isolation_rootfs` fails with signal 11 (SIGSEGV) when running PHP 8.5 -inside a chroot/rootfs-isolated Unit application. - -**Test:** `test/test_php_isolation.py` — skipped for PHP >= 8.5 with explicit reason. - -**Likely causes:** -- PHP 8.5 introduced new shared library dependencies not present in the minimal rootfs fixture -- Or PHP 8.5 accesses paths at startup that are unavailable in the chroot - -**Investigation steps:** -1. Run `ldd $(which php)` with PHP 8.5 and compare against the rootfs fixture contents -2. Check `unit.log` for the full path that caused the segfault (needs core dump or `strace`) -3. Check if `php 8.5 --define open_basedir=...` reproduces outside of Unit - ---- - ## Test Infrastructure ### Download statistics for packages.freeunit.org diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index 4d9eb21c1..f248da417 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -1,7 +1,4 @@ -import pytest - from unit.applications.lang.php import ApplicationPHP -from unit.option import option prerequisites = {'modules': {'php': 'any'}, 'features': {'isolation': True}} @@ -9,9 +6,6 @@ def test_php_isolation_rootfs(is_su, require, temp_dir): - versions = option.available['modules'].get('php', []) - if versions and versions[0].split('.')[:2] >= ['8', '5']: - pytest.skip('PHP 8.5 rootfs isolation: SIGSEGV under investigation') isolation = {'rootfs': temp_dir} if not is_su: diff --git a/test/unit/applications/lang/php.py b/test/unit/applications/lang/php.py index ac59ec1b4..568260eb3 100644 --- a/test/unit/applications/lang/php.py +++ b/test/unit/applications/lang/php.py @@ -1,10 +1,26 @@ +import os from pathlib import Path import shutil +import pytest + from unit.applications.proto import ApplicationProto from unit.option import option +def _absolute_symlinks(path, names): + # A shutil.copytree() filter. Relative symlinks stay inside the copy, + # but an absolute one -- /usr/share/zoneinfo/localtime -> /etc/localtime + # on Debian -- points back at the host, and the conftest teardown chmods + # every file it walks in the temp dir, following it out of the rootfs. + return [ + name + for name in names + if os.path.islink(f'{path}/{name}') + and os.path.isabs(os.readlink(f'{path}/{name}')) + ] + + class ApplicationPHP(ApplicationProto): def __init__(self, application_type='php'): self.application_type = application_type @@ -20,6 +36,27 @@ def load(self, script, index='index.php', **kwargs): if not Path(f'{rootfs}/app/php/{script}').exists(): shutil.copytree(script_path, f'{rootfs}/app/php/{script}') + # PHP built with --with-system-tzdata -- every Debian, Ubuntu and + # Fedora package -- opens the timezone database lazily, at request + # time, which is after Unit has chroot()ed the app worker into + # "rootfs". Such builds segfault instead of raising an error when + # the database is not there, so provision it and keep these tests + # about Unit rather than about PHP packaging. + tzdir = '/usr/share/zoneinfo' + + if not Path(tzdir).is_dir(): + pytest.skip( + f'no timezone database at {tzdir} to copy into the rootfs' + ) + + if not Path(f'{rootfs}{tzdir}').exists(): + shutil.copytree( + tzdir, + f'{rootfs}{tzdir}', + symlinks=True, + ignore=_absolute_symlinks, + ) + script_path = f'/app/php/{script}' app = { From ffcf291267e5c6b54404e21f9d89a4cddeecc7d4 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 18:02:05 +0200 Subject: [PATCH 073/391] fix(fuzzing): give the HTTP harnesses a log, so a logging handler is not a crash fuzz_http_h1p_peer segfaults about ten seconds in, on a request from its own seed corpus: SUMMARY: AddressSanitizer: SEGV src/nxt_http_proxy.c:432 in nxt_http_proxy_content_length Line 432 is the nxt_log() that accb5054 added when it started rejecting duplicate upstream Content-Length headers. The request the harness feeds to nxt_http_fields_process() is nxt_mp_zget() memory and nothing ever fills in req->task, so task->log is NULL and nxt_log() dereferences it. Any field handler that logs would do the same; the duplicate-Content-Length fix is merely the first one that does. So this is the harness, not the product: in a real router the request always carries a task with a log. Point req->task.log at nxt_main_log, the way nxt_http_controller_fuzz.c already points req->conn->log at it, and set the level to alert in LLVMFuzzerInitialize so a fuzzing run stays quiet. The request-side harness gets the same two lines. It has no logging handler today, but it builds its request exactly the same way, so it is the same crash waiting for the next handler that logs. With this, all five targets survive 60 seconds each; without it the peer target cannot be put in CI at all. Co-Authored-By: Claude Opus 5 --- fuzzing/nxt_http_h1p_fuzz.c | 10 ++++++++++ fuzzing/nxt_http_h1p_peer_fuzz.c | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/fuzzing/nxt_http_h1p_fuzz.c b/fuzzing/nxt_http_h1p_fuzz.c index 5b7f323ff..459941bef 100644 --- a/fuzzing/nxt_http_h1p_fuzz.c +++ b/fuzzing/nxt_http_h1p_fuzz.c @@ -28,6 +28,9 @@ LLVMFuzzerInitialize(int *argc, char ***argv) return NXT_ERROR; } + /* Keep a fuzzing run quiet: nothing below alert is worth printing. */ + nxt_main_log.level = NXT_LOG_ALERT; + ret = nxt_http_fields_hash(&nxt_h1p_fields_hash, nxt_h1p_fields, nxt_nitems(nxt_h1p_fields)); if (ret != NXT_OK) { @@ -76,6 +79,13 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) goto failed; } + /* + * A field handler may log, and nxt_log() dereferences task->log. The + * request is zeroed memory here, so without this any handler that logs + * is a null dereference in the harness rather than a finding. + */ + req->task.log = &nxt_main_log; + buf.start = (u_char *)data; buf.end = (u_char *)data + size; buf.pos = buf.start; diff --git a/fuzzing/nxt_http_h1p_peer_fuzz.c b/fuzzing/nxt_http_h1p_peer_fuzz.c index 79145ac78..6774286db 100644 --- a/fuzzing/nxt_http_h1p_peer_fuzz.c +++ b/fuzzing/nxt_http_h1p_peer_fuzz.c @@ -28,6 +28,9 @@ LLVMFuzzerInitialize(int *argc, char ***argv) return NXT_ERROR; } + /* Keep a fuzzing run quiet: nothing below alert is worth printing. */ + nxt_main_log.level = NXT_LOG_ALERT; + ret = nxt_http_fields_hash(&nxt_h1p_peer_fields_hash, nxt_h1p_peer_fields, nxt_nitems(nxt_h1p_peer_fields)); @@ -72,6 +75,13 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) goto failed; } + /* + * A field handler may log, and nxt_log() dereferences task->log. The + * request is zeroed memory here, so without this any handler that logs + * is a null dereference in the harness rather than a finding. + */ + req->task.log = &nxt_main_log; + buf.start = (u_char *)data; buf.end = (u_char *)data + size; buf.pos = buf.start; From 544165e02f198f8ff171cdb658aa55ce779bd025 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 18:05:45 +0200 Subject: [PATCH 074/391] ci(fuzzing): run the in-repo fuzzers on pull requests The fuzzers have been in the tree since 2022 and no pull request has ever run one. The workflow that was meant to has been workflow_dispatch-only since 8d59a464 (April 2026), which records no reason -- its subject is just the line it changed and it has no body -- and by the January note in 148cc0b1 it had "never run at all". It could not have worked from here in any case. The OSS-Fuzz project it invokes is archived -- projects/unit/project.yaml carries `disabled: true` and the comment "Project is archived" -- and its `main_repo` is https://github.com/nginx/unit, so CIFuzz would have built upstream's source rather than the pull request's. Fetched from google/oss-fuzz master today. Build the in-repo targets instead: clang with ASan and UBSan, the tree's own `make fuzz`, and fuzzing/run-ci.sh to give each target a bounded budget against its checked-in seed corpus and dictionary. A pull request touching src/nxt_http*, src/nxt_h1proto*, src/nxt_controller*, src/nxt_conf*, fuzzing/, configure or auto/ runs all five targets for 60 seconds each; a manual dispatch takes a budget and a target list. Reproducers are uploaded on failure, because a fuzzing failure without one is not actionable. nxt_controller.c is in the path list because nxt_http_controller_fuzz.c #includes it: a change there is a change to what this job fuzzes. configure and auto/ are there because --fuzz, auto/fuzzing and auto/sources are what produce the binaries at all. Running a subset per changed path was tried and dropped. Every filter above already maps to a target and fuzzing/** maps to all of them, so a target left out of the list is only a harness CI compiles and never runs. Two settings are what make this a gate rather than a formality, and both were arrived at by measurement. halt_on_error=1, because -fsanitize=undefined is recoverable by default: UBSan prints the report, execution continues, libFuzzer exits 0 and the check is green on a finding. Reproduced on a purpose-built target before fixing it. It is appended to UBSAN_OPTIONS rather than defaulted into it -- sanitizer flags are parsed left to right and the last wins, so it stays authoritative even against a caller's own halt_on_error=0. Not print_stacktrace=1 as well, though: that pulls in llvm-symbolizer, which is the second setting's whole problem. -print_funcs=0, because libFuzzer symbolizes every newly covered function to print its NEW_FUNC line, and llvm-symbolizer takes about thirteen seconds a call against a 14 MB ASan+UBSan binary. On a fresh corpus, which is what CI has every time, that is the entire run. At a 30-second budget: fuzz_json 7 runs / 90s -> 1190884 runs / 31s fuzz_http_h1p 161 runs / 91s -> 1267717 runs / 31s Crash reports keep their symbols either way; those come from the sanitizer's stack printer, not from libFuzzer's coverage output. All five targets at 60 seconds now take 5m07s in total and execute between two and three and a half million inputs each, with no findings on master. -max_total_time is still only checked between runs, and libFuzzer's own -timeout defaults to twenty minutes, so each target runs under a wall-clock cap of its budget plus a minute. Hitting the cap warns rather than fails: slowness is not a finding, and a red check for it would only teach people to ignore the job. That warning is not extended to every kill, though -- `timeout -k` reports 137 for a target it had to SIGKILL and so does the kernel's OOM killer, and an OOM is a finding. The elapsed time separates them: a kill at or past the cap is the cap, a kill well inside it fails. The job timeout follows the trigger. A pull request gets 25 minutes, because a hung job must not hold a runner for an hour; a dispatch, which can ask for a long budget across all five targets, gets 120. A missing binary or corpus is checked before anything runs, and the summary counts targets attempted rather than targets that succeeded: a build that silently produced no fuzzers, or an empty target list, would otherwise be a green check that fuzzed nothing. Co-Authored-By: Claude Opus 5 --- .github/workflows/fuzzing.yml | 138 ++++++++++++++++++-------- fuzzing/README.md | 45 +++++++++ fuzzing/run-ci.sh | 180 ++++++++++++++++++++++++++++++++++ 3 files changed, 324 insertions(+), 39 deletions(-) create mode 100755 fuzzing/run-ci.sh diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index 8a3425598..6d37ab1ee 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -1,43 +1,103 @@ -name: "Fuzzing (OSS-Fuzz)" +name: "Fuzzing (libFuzzer)" + on: - workflow_dispatch: # manual only - # pull_request: - # paths: - # - 'src/**' - # - 'fuzzing/**' - # - '.github/workflows/fuzzing.yml' - -permissions: {} + # The HTTP parsers are the reason this runs on pull requests: they are the + # code an attacker reaches first, and the seed corpus already covers them. + pull_request: + paths: + - 'src/nxt_http*' + - 'src/nxt_h1proto*' + - 'src/nxt_conf*' + # nxt_http_controller_fuzz.c #includes nxt_controller.c, so a change + # there is a change to what this job fuzzes. + - 'src/nxt_controller*' + # configure --fuzz, auto/fuzzing and auto/sources are what produce the + # binaries; break them and the job has nothing to run. + - configure + - 'auto/**' + - 'fuzzing/**' + - '.github/workflows/fuzzing.yml' + workflow_dispatch: + inputs: + seconds: + description: 'libFuzzer budget per target, in seconds' + required: false + default: '60' + targets: + description: 'Space-separated fuzzer names (empty = all five)' + required: false + default: '' + +permissions: + contents: read + +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a dispatch is never grouped with + # another: Actions keeps only one PENDING run per group and would drop a + # queued one even with cancel-in-progress false. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: - Fuzzing: + libfuzzer: + name: libFuzzer (ASan + UBSan) runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - security-events: write + # Five targets at a 60s budget is a little over five minutes -- each + # finishes within a second or two of its budget, and run-ci.sh caps the + # worst case at two minutes each. The clang/ASan build is a few minutes + # on top. A dispatch can ask for a much longer budget across all five + # targets, so it gets a much longer bound; a pull request must not, or a + # hung job holds a runner for an hour. + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 120 || 25 }} + steps: - - name: Build Fuzzers - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: 'unit' - language: c - - name: Run Fuzzers - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - oss-fuzz-project-name: 'unit' - language: c - fuzz-seconds: 300 - output-sarif: true - - name: Upload Crash - uses: actions/upload-artifact@v4 - if: failure() && steps.build.outcome == 'success' - with: - name: artifacts - path: ./out/artifacts - - name: Upload Sarif - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v4 - with: - # Path to SARIF file relative to the root of the repository - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + - uses: actions/checkout@v5 + + # libclang-rt-dev carries libclang_rt.fuzzer, which -fsanitize=fuzzer + # links against; clang alone is not enough. + - name: Install clang and the sanitizer runtimes + run: | + sudo apt-get -qq update + sudo apt-get -qq install -y --no-install-recommends \ + clang llvm libclang-rt-dev + + - name: Build the fuzzers + env: + CC: clang + CXX: clang++ + CFLAGS: >- + -g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + -fsanitize=address,undefined -fsanitize=fuzzer-no-link + CXXFLAGS: >- + -g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + -fsanitize=address,undefined -fsanitize=fuzzer-no-link + # configure compiles and runs small probe programs. Built with ASan + # they report leaks and exit non-zero, and configure then records the + # feature as absent -- a silently different build, not a failure. + ASAN_OPTIONS: detect_leaks=0 + run: | + ./configure --no-regex --no-pcre2 --fuzz=-fsanitize=fuzzer + make fuzz -j"$(nproc)" + + # All five targets, on a pull request and on a dispatch alike. Picking a + # subset per changed path was tried and dropped: every path filter above + # already maps to a target, `fuzzing/**` maps to all of them, and a + # target left out of the list is a harness that CI compiles but never + # runs. At about 61 seconds each the whole set is close to five minutes. + - name: Run the fuzzers + env: + FUZZ_SECONDS: ${{ github.event.inputs.seconds || '60' }} + FUZZ_TARGETS: ${{ github.event.inputs.targets }} + run: | + # shellcheck disable=SC2086 # deliberate word splitting + ./fuzzing/run-ci.sh -t "${FUZZ_SECONDS}" ${FUZZ_TARGETS} + + # A reproducer is the only thing that makes a fuzzing failure actionable. + - name: Upload reproducers + if: failure() + uses: actions/upload-artifact@v4 + with: + name: fuzz-artifacts + path: build/fuzz-artifacts/ + if-no-files-found: ignore diff --git a/fuzzing/README.md b/fuzzing/README.md index 9c70c8016..c0be7aecc 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -66,3 +66,48 @@ may vary). ``` clang, llvm & compiler-rt ``` + +## In CI + +`.github/workflows/fuzzing.yml` builds these targets with clang, ASan and +UBSan and runs `fuzzing/run-ci.sh`, which gives each target a bounded +libFuzzer budget against its seed corpus and fails on the first crash. + +```shell +$ fuzzing/run-ci.sh # all five, 60s each +$ fuzzing/run-ci.sh -t 120 fuzz_http_h1p # one target, longer +``` + +A pull request that touches `src/nxt_http*`, `src/nxt_h1proto*`, +`src/nxt_controller*`, `src/nxt_conf*` or `fuzzing/` runs all five targets -- +roughly five minutes; a manual `workflow_dispatch` takes a budget and a +target list as inputs. Reproducers land in `build/fuzz-artifacts/` and are +uploaded as an artifact when the job fails. + +Each target finishes within a second or two of its budget. `run-ci.sh` caps +it at the budget plus a minute anyway -- libFuzzer only checks the budget +between runs -- and treats hitting that cap as a warning rather than a +failure, while distinguishing it from a kill that arrives early: an OOM is a +finding, not slowness. + +It sets `UBSAN_OPTIONS=halt_on_error=1`. `-fsanitize=undefined` is +recoverable by default -- UBSan prints the report and carries on, libFuzzer +exits 0, and the check goes green on a finding. It deliberately does *not* +set `print_stacktrace=1`, which reintroduces the symbolizer cost below; the +report keeps its file:line and the reproducer is saved, so rerun the +reproducer with `UBSAN_OPTIONS=print_stacktrace=1` when you want the trace. + +It also passes `-print_funcs=0`, which is worth knowing about if you run the +fuzzers by hand. libFuzzer symbolizes every newly covered function to print +its `NEW_FUNC` line, and llvm-symbolizer needs about thirteen seconds per call +against a 14 MB ASan+UBSan binary. On a fresh corpus -- which is what CI has +every time -- that is the whole run: `fuzz_json` managed **7 executions in 90 +seconds** with the default, and **1,190,884 in 31 seconds** without it. Crash +reports keep their symbols either way; those come from the sanitizer's stack +printer, not from libFuzzer's coverage output. + +This is a smoke gate, not a campaign. It replaced an OSS-Fuzz CIFuzz job +that could not have worked here: the OSS-Fuzz `unit` project is marked +`disabled: true` ("Project is archived") and its `main_repo` is +`https://github.com/nginx/unit`, so a run from this repository would have +built upstream's source rather than the pull request's. diff --git a/fuzzing/run-ci.sh b/fuzzing/run-ci.sh new file mode 100755 index 000000000..50c37bb58 --- /dev/null +++ b/fuzzing/run-ci.sh @@ -0,0 +1,180 @@ +#!/bin/sh +# +# Run libFuzzer targets for a bounded time against their seed corpora and fail +# on the first crash. This is a smoke gate, not a fuzzing campaign: the +# question it answers is "does the HTTP parser still survive its own seed +# corpus and a minute of shallow mutation", which is the part worth blocking a +# pull request on. +# +# Usage: +# fuzzing/run-ci.sh [-t SECONDS] [TARGET ...] +# +# -t SECONDS libFuzzer budget per target (default 60) +# TARGET ... fuzzer names; default is all five +# +# Expects the fuzzers to be built already: +# CC=clang CXX=clang++ \ +# CFLAGS="-g -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION \ +# -fsanitize=address,undefined -fsanitize=fuzzer-no-link" \ +# ./configure --no-regex --no-pcre2 --fuzz=-fsanitize=fuzzer && make fuzz +# +# Any crash, leak or sanitizer report leaves the reproducer in +# build/fuzz-artifacts/ for CI to upload. + +set -eu + +BUDGET=60 +BUILD=${BUILD_DIR:-build} + +die() { echo "run-ci.sh: $*" >&2; exit 1; } + +while getopts 't:' opt; do + case "$opt" in + t) BUDGET=$OPTARG ;; + *) die "usage: $0 [-t SECONDS] [TARGET ...]" ;; + esac +done +shift $((OPTIND - 1)) + +# The budget reaches this from a workflow input and is then used in shell +# arithmetic, where a non-numeric value is at best a confusing error. +case "$BUDGET" in + ''|*[!0-9]*) die "budget must be a whole number of seconds, got '$BUDGET'" ;; +esac + +# Seed corpus and dictionary per target. fuzz_http_* share the HTTP corpus. +corpus_for() { + case "$1" in + fuzz_basic) echo fuzz_basic_seed_corpus ;; + fuzz_json) echo fuzz_json_seed_corpus ;; + fuzz_http_*) echo fuzz_http_seed_corpus ;; + *) die "unknown target: $1" ;; + esac +} + +dict_for() { + case "$1" in + fuzz_http_*) echo fuzz_http.dict ;; + *) echo "" ;; + esac +} + +[ $# -gt 0 ] || set -- fuzz_basic fuzz_json fuzz_http_controller \ + fuzz_http_h1p fuzz_http_h1p_peer + +# Freeze the list: the loop below rebuilds "$@" to hold each target's libFuzzer +# arguments, so it cannot also be the list being iterated. +TARGETS=$* + +# -fsanitize=undefined is recoverable by default: UBSan prints the report and +# execution continues, so libFuzzer exits 0 and a green check hides the +# finding. halt_on_error turns a report into an abort, which libFuzzer +# records as a crash and saves a reproducer for. Set here rather than in the +# build flags so the guarantee travels with the runner, whoever built the +# binaries. +# +# Deliberately without print_stacktrace=1: that pulls in llvm-symbolizer, the +# same thirteen-seconds-a-call cost that -print_funcs=0 exists to avoid, and +# it turns the abort into a stall the wall-clock cap then files as "slow". +# The report keeps its file:line and the reproducer is saved, which is what a +# rerun needs. +# Appended rather than defaulted: sanitizer flags are parsed left to right and +# the last wins, so this stays authoritative even when the caller already has +# UBSAN_OPTIONS set to something (including halt_on_error=0). +UBSAN_OPTIONS="${UBSAN_OPTIONS:+$UBSAN_OPTIONS:}halt_on_error=1" +export UBSAN_OPTIONS + +ART="$BUILD/fuzz-artifacts" +CORPORA="$BUILD/fuzz-corpora" +mkdir -p "$ART" "$CORPORA" || die "cannot write under $BUILD -- wrong BUILD_DIR?" + +# Check every binary up front: a build that silently produced nothing must not +# be mistaken for a clean run. +for target in $TARGETS; do + [ -x "$BUILD/$target" ] || die "no fuzzer at $BUILD/$target -- was 'make fuzz' run?" + [ -d "fuzzing/$(corpus_for "$target")" ] \ + || die "no seed corpus fuzzing/$(corpus_for "$target") for $target" +done + +# -print_funcs=0 is not cosmetic. libFuzzer symbolizes every newly covered +# function to print its NEW_FUNC line, and llvm-symbolizer takes about thirteen +# seconds per call against a 14 MB ASan+UBSan binary, so the exploration phase +# -- exactly the part a fresh CI corpus is all of -- stalls on the symbolizer. +# Measured here, 30-second budget, before and after: +# +# fuzz_json 7 runs / 90s -> 1190884 runs / 31s +# fuzz_http_h1p 161 runs / 91s -> 1267717 runs / 31s +# fuzz_http_controller -> 1567704 runs / 31s +# fuzz_http_h1p_peer -> 1442178 runs / 32s +# +# Crash reports keep their symbols either way: those go through the sanitizer's +# own stack printer, not libFuzzer's coverage output. Verified on a purpose- +# built crashing target. +PRINT_FUNCS=-print_funcs=0 + +# -max_total_time is only checked between runs and libFuzzer's own -timeout +# defaults to twenty minutes, so a single pathological unit can still run far +# past the budget. Bound the wall clock rather than tightening -timeout: a +# slow target is not a finding, and turning slowness into a red check would +# only teach people to ignore the job. A minute of headroom is ample now that +# a target finishes within a second or two of its budget. +# +# FUZZ_WALL_MARGIN exists so the two branches below can be exercised in a +# test without waiting a minute for the cap. +WALL=$((BUDGET + ${FUZZ_WALL_MARGIN:-60})) + +status=0 +attempted=0 +ran=0 +slow=0 + +for target in $TARGETS; do + seed=$(corpus_for "$target") + dict=$(dict_for "$target") + + mkdir -p "$CORPORA/$target" + set -- "$CORPORA/$target" "fuzzing/$seed" \ + "-max_total_time=$BUDGET" \ + "-artifact_prefix=$ART/$target-" \ + "$PRINT_FUNCS" -print_final_stats=1 -rss_limit_mb=2560 + [ -n "$dict" ] && set -- "$@" "-dict=fuzzing/$dict" + + attempted=$((attempted + 1)) + echo "=== $target (${BUDGET}s budget, ${WALL}s wall cap, seeds: $seed${dict:+, dict: $dict}) ===" + rc=0 + started=$(date +%s) + timeout -k 5 "$WALL" "$BUILD/$target" "$@" || rc=$? + elapsed=$(( $(date +%s) - started )) + + # 137 is 128+SIGKILL, which is what `timeout -k` reports for a target it + # had to kill -- and also what the kernel's OOM killer produces mid-run. + # Distinguishing them matters: an OOM is a finding and must not be filed + # under "slow". Only a kill at or past the cap is the cap. + case "$rc" in + 0) + ran=$((ran + 1)) + ;; + 124|137) + if [ "$elapsed" -ge "$WALL" ]; then + echo "run-ci.sh: $target hit the ${WALL}s wall cap (not a finding)" >&2 + slow=$((slow + 1)) + ran=$((ran + 1)) + else + echo "run-ci.sh: $target KILLED after ${elapsed}s, well inside the" \ + "${WALL}s cap -- treating as a failure (OOM killer?)" >&2 + status=1 + fi + ;; + *) + echo "run-ci.sh: $target FAILED (exit $rc after ${elapsed}s)" >&2 + status=1 + ;; + esac + echo "" +done + +# Guard on what was attempted, not on what succeeded: an empty target list +# would otherwise be a green run that fuzzed nothing. +[ "$attempted" -gt 0 ] || die "no fuzzer actually ran" +echo "run-ci.sh: $attempted target(s) attempted, $ran finished, $slow hit the wall cap, ${BUDGET}s budget each" +exit "$status" From 1286b0d778fd1789cb4bdd5eb8fe9e98c759b7d3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 17:17:43 +0200 Subject: [PATCH 075/391] ci: gate on syscall drift, on glibc and on musl Unit's portability story is a list of syscalls. The kernel minimum, what a container's seccomp profile has to allow, and what a musl build does differently from a glibc one are all claims about that list, and nothing kept them honest -- a 2026-07 audit of it had to be assembled by hand, and several of its conclusions were already stale by the time it was written. Capture the list instead. test/syscalls/capture.sh runs unitd under strace -f: PUT a configuration with one listener and one static share route, fetch a file over it, assert 200 and the body, SIGQUIT, then reduce the trace to sorted unique syscall names. Startup, configuration, a served request and teardown are all in one trace, and because the container runs it as root the router drops privileges inside it too, so setgid and setuid are covered. Not capget, and the distinction matters because the audit assumed otherwise: nxt_capability_set() returns at src/nxt_capability.c:51 as soon as geteuid() == 0, so the capget below it is only reached by a non-root daemon and appears in neither baseline. A non-root leg would cover it and is worth adding later. New syscalls fail the job; disappeared ones only warn. Reaching for a new kernel interface moves the kernel minimum or needs a seccomp allowance, so it belongs in review with the baseline updated in the same commit. Losing one is never a hazard and a libc update does it unprompted. The job is advisory to begin with -- continue-on-error, and "(advisory)" in its name. The container pins the libc by digest, but which syscalls glibc picks still depends partly on the host kernel, and these baselines were captured on a developer machine: a runner on an older kernel could legitimately add a fallback syscall the baseline has never seen, and a red gate that is more likely wrong than right teaches people to ignore it. Every run uploads its captured list as an artifact, including failed runs, so promoting the job is: read the artifact, commit it, drop two lines. test/syscalls/README.md spells that out. Both legs run inside a digest-pinned image -- debian:trixie-slim and alpine:3.22 -- rather than a moving tag, so a baseline cannot drift with no change in this repository, and a developer reproduces exactly what CI saw with one command. The digest does not pin the packages installed on top of it; both are stable releases, so that only moves on a point release, but it is a real residual and another reason to start advisory. The two baselines differ by 11 entries one way and 6 the other -- epoll_create vs epoll_create1, clone3 vs fork, openat vs open, memfd_create only on glibc -- which is the whole reason for capturing both. No language application is started, deliberately: the request served is a static share route, with no module and no prototype process. An app process is forked from Unit and never execs, so in the trace its syscalls are Unit's; a CPython or PHP interpreter contributes more of them than Unit does, all of it moving with the runtime's version. The gate would fail on a language patch release, which is noise rather than drift. An empty list compares clean against any baseline, so both scripts refuse a capture under 20 syscalls and capture.sh additionally demands seven Unit cannot avoid. It also fails on a nonzero exit from the traced daemon: strace exits with its tracee's status, so a daemon that crashed during its SIGQUIT teardown would otherwise hand over a large, plausible list with every unreached teardown syscall filed as a harmless warning. Every sort is LC_ALL=C: comm rejects input collated under a different locale, and '_' is exactly where locales disagree. strace runs with --kill-on-exit so a killed tracer takes the Unit process family with it rather than detaching and leaving the router on the test port, every curl has a deadline, and every kill in the cleanup trap is best-effort -- under set -e a failing kill would abort the trap before it removed the temporary directory. Co-Authored-By: Claude Opus 5 --- .github/workflows/security-syscalls.yml | 77 +++++++++ test/syscalls/README.md | 124 ++++++++++++++ test/syscalls/baseline-linux-glibc.txt | 71 ++++++++ test/syscalls/baseline-linux-musl.txt | 66 ++++++++ test/syscalls/capture.sh | 206 ++++++++++++++++++++++++ test/syscalls/check.sh | 66 ++++++++ test/syscalls/run-in-docker.sh | 126 +++++++++++++++ 7 files changed, 736 insertions(+) create mode 100644 .github/workflows/security-syscalls.yml create mode 100644 test/syscalls/README.md create mode 100644 test/syscalls/baseline-linux-glibc.txt create mode 100644 test/syscalls/baseline-linux-musl.txt create mode 100755 test/syscalls/capture.sh create mode 100755 test/syscalls/check.sh create mode 100755 test/syscalls/run-in-docker.sh diff --git a/.github/workflows/security-syscalls.yml b/.github/workflows/security-syscalls.yml new file mode 100644 index 000000000..a30823378 --- /dev/null +++ b/.github/workflows/security-syscalls.yml @@ -0,0 +1,77 @@ +name: "Security (Syscall Drift)" + +on: + pull_request: + paths: + - configure + - 'auto/**' + - 'src/**' + - 'test/syscalls/**' + - '.github/workflows/security-syscalls.yml' + push: + branches: master + paths: + - configure + - 'auto/**' + - 'src/**' + - 'test/syscalls/**' + - '.github/workflows/security-syscalls.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + syscalls: + name: Syscall drift, advisory (${{ matrix.libc }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + + # Advisory until a baseline has been captured from a CI run and committed. + # The container pins the libc by digest, but which syscalls glibc picks + # still depends partly on the host kernel, and these baselines were taken + # on a developer machine -- so a runner on an older kernel could add a + # fallback syscall the baseline has never seen, and a red gate here would + # more likely be wrong than right. test/syscalls/README.md has the + # four-step sequence for making it blocking; it is one commit. + continue-on-error: true + + strategy: + fail-fast: false + matrix: + libc: [ glibc, musl ] + + steps: + - uses: actions/checkout@v5 + + # Everything happens inside a pinned container (debian:trixie-slim / + # alpine:3.22), so the runner image can move without moving the baseline: + # the captured set depends on the libc, not on the host. The runner's own + # kernel is still in play -- see the artifact note below. + - name: Capture the syscall set + run: test/syscalls/run-in-docker.sh ${{ matrix.libc }} + -o "syscalls-${{ matrix.libc }}.txt" + + # Uploaded even on failure: reconciling a baseline against what CI + # actually observed is the documented update path (test/syscalls/README.md) + # and it only works if the list survives a red run. + - name: Upload the captured list + if: always() + uses: actions/upload-artifact@v4 + with: + name: syscalls-${{ matrix.libc }} + path: syscalls-${{ matrix.libc }}.txt + if-no-files-found: warn + + - name: Compare against the baseline + run: test/syscalls/check.sh + "test/syscalls/baseline-linux-${{ matrix.libc }}.txt" + "syscalls-${{ matrix.libc }}.txt" diff --git a/test/syscalls/README.md b/test/syscalls/README.md new file mode 100644 index 000000000..800d51c2a --- /dev/null +++ b/test/syscalls/README.md @@ -0,0 +1,124 @@ +# Syscall drift gate + +A CI gate that notices when Unit starts using a kernel interface it did not use +before. It exists because Unit's portability story is a list of syscalls: the +kernel minimum, what a container's seccomp profile has to allow, and what a +`musl` build does differently from a `glibc` one are all statements about this +list, and nothing kept them honest. + + test/syscalls/capture.sh run unitd under strace, print its syscalls + test/syscalls/check.sh diff a capture against a baseline + test/syscalls/run-in-docker.sh do both inside a pinned container + test/syscalls/baseline-linux-glibc.txt + test/syscalls/baseline-linux-musl.txt + +`.github/workflows/security-syscalls.yml` runs the docker driver for both libcs +on every pull request that touches the build or `src/`. + +## The job is advisory for now + +The workflow is marked `continue-on-error: true` and its jobs are named +"(advisory)": a difference shows up in the checks list and in the log, but it +cannot block a merge. + +That is not timidity, it is honesty about where the baselines came from. They +were captured on a developer machine, and while the container pins the libc by +digest, the syscalls glibc picks still depend partly on the **host kernel** -- +so a runner on an older kernel can legitimately add a fallback syscall the +baseline has never seen. Until a baseline has been captured from a CI run and +committed, a red gate here would be more likely to be wrong than right. + +**Making it blocking** is one commit, and this is the sequence: + +1. Let the workflow run on a pull request (or dispatch it). +2. Download the `syscalls-glibc` and `syscalls-musl` artifacts -- they are + uploaded on every run, including failures, for exactly this. +3. Commit them as the baselines if they differ from what is here, noting in + the commit message which syscalls moved and why. +4. Drop `continue-on-error` and the "(advisory)" suffix. + +## The rule + +**A new syscall fails the job. A disappeared syscall only warns.** + +Reaching for a new kernel interface is a decision -- it moves the kernel +minimum, or needs a seccomp allowance, or does not exist on some platform -- so +it should be visible in review, with the baseline updated in the same commit. +Dropping one is never a hazard, and a libc or runner update drops one now and +then without anybody meaning it to. + +## Updating a baseline + +```sh +test/syscalls/run-in-docker.sh glibc --update +test/syscalls/run-in-docker.sh musl --update +git add test/syscalls/baseline-linux-*.txt +``` + +Commit the refreshed baseline together with the change that caused it, and say +in the commit message which syscall appeared and why. If the capture differs +from what CI sees, take CI's: the workflow uploads its captured list as the +`syscalls-` artifact on every run, exactly so a baseline can be +reconciled without guessing. + +Bumping either pinned image digest in `run-in-docker.sh` is also a baseline +refresh. + +## What is exercised, and what is not + +`capture.sh` starts `unitd`, PUTs a configuration with one TCP listener and one +`share` route, fetches a static file over it, asserts HTTP 200 and the expected +body, then stops the daemon with `SIGQUIT` -- all inside `strace -f`, so +startup, configuration, request handling and teardown are in one trace. The +served request is a static `share` route and nothing else: no application, no +language module, no prototype process. Run as root (which is how it runs in +the container) the router drops to the unprivileged default user, so `setgid` +and `setuid` are in the trace. + +`capget` is **not**, and it is worth saying why, because the audit that +prompted this gate assumed otherwise. `nxt_capability_set()` returns at +`src/nxt_capability.c:51` as soon as `geteuid() == 0` -- root already has what +it needs -- so the `capget` call below it is only reached by a non-root daemon. +Neither baseline contains it. A non-root capture leg would, and is listed as +a follow-up. + +No language application is started, on purpose. An app process is forked from +Unit and never `exec`s, so its syscalls are indistinguishable in the trace from +Unit's own -- and a CPython or PHP interpreter contributes far more of them +than Unit does, all of it moving with the runtime's own version. The gate +would then fail on a language patch release, which is noise, not drift. +Covering the module path is a separate job with a separate baseline; see the +follow-ups in the pull request that added this. + +The two lists are not directly comparable with each other: 11 syscalls appear +only in the glibc baseline and 6 only in the musl one, because the libcs pick +different syscalls for the same operation (`epoll_create` vs `epoll_create1`, +`clone3` vs `fork`, `openat` vs `open`, and `memfd_create` only on glibc). +That is the reason both are captured. + +## Guards + +Both scripts refuse a capture with fewer than 20 syscalls, and `capture.sh` +additionally requires a handful of syscalls Unit cannot possibly avoid +(`epoll_ctl`, `socket`, `bind`, `listen`, `read`, `write`, `close`). A missing +`strace`, a changed output format or a parse that quietly matched nothing all +produce an empty list, and an empty list compares clean against any baseline -- +a green check that verified nothing. + +`capture.sh` also fails on a nonzero exit from the traced daemon. `strace` +exits with its tracee's status, so a daemon that crashes during its `SIGQUIT` +teardown would otherwise still hand over a large, plausible list, and every +syscall the teardown never reached would be filed as a harmless warning. + +## Requirements + +`capture.sh` needs `strace` and `curl` and a built tree. `run-in-docker.sh` +needs only docker; it installs a toolchain, builds Unit and captures inside a +digest-pinned container. The digest pins the base image and therefore the +libc, but not the packages installed on top of it -- both are stable releases, +so that moves only on a point release, but it is a real residual. It passes `--cap-add=SYS_PTRACE`: on a stock daemon strace can +already trace its own descendants without it (verified on docker 29.7.2 with +the default seccomp and AppArmor profiles), but a host with a tightened +`ptrace_scope` or a custom profile needs it, and it costs nothing here. The +image is always pulled `--platform linux/amd64`: the baselines are per +architecture and an arm64 pull would silently produce a different list. diff --git a/test/syscalls/baseline-linux-glibc.txt b/test/syscalls/baseline-linux-glibc.txt new file mode 100644 index 000000000..aca252e5b --- /dev/null +++ b/test/syscalls/baseline-linux-glibc.txt @@ -0,0 +1,71 @@ +# Syscalls issued by unitd on linux/amd64, glibc. +# +# Regenerate: test/syscalls/run-in-docker.sh glibc --update +# Image: debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132 +# See test/syscalls/README.md before editing this by hand. +accept4 +access +arch_prctl +bind +brk +chmod +clone +clone3 +close +connect +dup2 +epoll_create +epoll_ctl +epoll_wait +eventfd2 +execve +exit_group +fcntl +fstat +ftruncate +getdents64 +getegid +geteuid +getpid +getppid +getrandom +getsockopt +gettid +ioctl +listen +lseek +memfd_create +mkdir +mmap +mprotect +munmap +newfstatat +openat +prctl +pread64 +prlimit64 +pwrite64 +read +recvfrom +recvmsg +rename +rseq +rt_sigaction +rt_sigprocmask +sched_getaffinity +sendmsg +set_robust_list +set_tid_address +setgid +setgroups +setsockopt +setuid +shutdown +signalfd4 +socket +socketpair +uname +unlink +wait4 +write +writev diff --git a/test/syscalls/baseline-linux-musl.txt b/test/syscalls/baseline-linux-musl.txt new file mode 100644 index 000000000..6caed05d6 --- /dev/null +++ b/test/syscalls/baseline-linux-musl.txt @@ -0,0 +1,66 @@ +# Syscalls issued by unitd on linux/amd64, musl. +# +# Regenerate: test/syscalls/run-in-docker.sh musl --update +# Image: alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce +# See test/syscalls/README.md before editing this by hand. +accept4 +arch_prctl +bind +brk +chmod +clone +close +connect +dup2 +epoll_create1 +epoll_ctl +epoll_pwait +eventfd2 +execve +exit_group +fcntl +fork +fstat +ftruncate +futex +getdents64 +getegid +geteuid +getpid +getppid +getrandom +getsockopt +gettid +ioctl +listen +membarrier +mkdir +mmap +mprotect +munmap +open +prctl +pread64 +pwrite64 +read +recvfrom +recvmsg +rename +rt_sigaction +rt_sigprocmask +sched_getaffinity +sendmsg +set_tid_address +setgid +setgroups +setsockopt +setuid +shutdown +signalfd4 +socket +socketpair +uname +unlink +wait4 +write +writev diff --git a/test/syscalls/capture.sh b/test/syscalls/capture.sh new file mode 100755 index 000000000..3f5d11a0a --- /dev/null +++ b/test/syscalls/capture.sh @@ -0,0 +1,206 @@ +#!/bin/sh +# +# Capture the set of syscalls unitd issues while starting up, accepting a +# configuration and serving one request, and print it as a sorted, unique list +# of syscall names -- one per line. +# +# The list is the input to the CI drift gate (test/syscalls/check.sh); see +# test/syscalls/README.md for the whole story, including why the exercised +# configuration deliberately does not start a language application. +# +# Usage: +# test/syscalls/capture.sh [-u UNITD] [-p PORT] [-o OUTFILE] +# +# -u path to the unitd binary (default: build/sbin/unitd) +# -p TCP port for the test listener (default: 8099) +# -o write the list here instead of stdout +# +# Requires: strace, curl. Run it from the top of a built source tree. + +set -eu + +UNITD=build/sbin/unitd +PORT=8099 +OUT= + +while getopts 'u:p:o:' opt; do + case "$opt" in + u) UNITD=$OPTARG ;; + p) PORT=$OPTARG ;; + o) OUT=$OPTARG ;; + *) echo "usage: $0 [-u UNITD] [-p PORT] [-o OUTFILE]" >&2; exit 2 ;; + esac +done + +die() { echo "capture.sh: $*" >&2; exit 1; } + +command -v strace >/dev/null 2>&1 || die "strace not found" +command -v curl >/dev/null 2>&1 || die "curl not found" +[ -x "$UNITD" ] || die "no unitd binary at $UNITD" + +WORK=$(mktemp -d) +STRACE_PID= + +# strace sets SIGINT/SIGQUIT/SIGTERM/SIGHUP to SIG_IGN in itself so that a +# terminal signal reaches the tracee instead, which means the daemon has to be +# signalled directly -- signalling strace is a no-op that hangs the script. +# unitd writes its main pid to --pid, so read it from there. +unit_pid() { + [ -s "$WORK/unit.pid" ] && cat "$WORK/unit.pid" +} + +# Every kill here is best-effort: by the time cleanup runs the daemon has +# usually exited already, and under `set -e` a failing kill would abort the +# EXIT trap before the rm and leak the temporary directory. +cleanup() { + up=$(unit_pid || :) + [ -n "$up" ] && { kill -QUIT "$up" 2>/dev/null || :; } + i=0 + while [ "$i" -lt 50 ] \ + && [ -n "$STRACE_PID" ] && kill -0 "$STRACE_PID" 2>/dev/null; do + i=$((i + 1)) + sleep 0.1 + done + [ -n "$up" ] && { kill -KILL "$up" 2>/dev/null || :; } + # SIGKILL on strace leaves its tracees behind by default -- hence + # --kill-on-exit below, so the controller and router go with it and do not + # sit on the test port. + [ -n "$STRACE_PID" ] && { kill -KILL "$STRACE_PID" 2>/dev/null || :; } + rm -rf "$WORK" + return 0 +} +trap cleanup EXIT INT TERM + +mkdir -p "$WORK/state" "$WORK/tmp" "$WORK/modules" "$WORK/share" +echo 'syscall drift canary' > "$WORK/share/index.txt" + +# Started as root, unitd drops the router to the configured unprivileged user +# (that is the point: setgid and setuid land in the trace). mktemp -d makes +# the directory 0700, so the router would then get EACCES and answer 403. +chmod 711 "$WORK" +chmod -R a+rX "$WORK/share" + +SOCK=$WORK/control.sock +TRACE=$WORK/trace.log + +cat > "$WORK/config.json" <"$WORK/stdout.log" 2>&1 & +STRACE_PID=$! + +# Wait for the control socket rather than sleeping a fixed amount: a slow +# runner must not silently produce a short trace. +i=0 +while [ ! -S "$SOCK" ]; do + i=$((i + 1)) + [ "$i" -lt 300 ] || { + cat "$WORK/stdout.log" >&2 || : + die "control socket did not appear within 30s" + } + kill -0 "$STRACE_PID" 2>/dev/null || { + cat "$WORK/stdout.log" >&2 || : + die "unitd exited before the control socket appeared" + } + sleep 0.1 +done + +curl -sS --fail --max-time 10 --unix-socket "$SOCK" \ + -X PUT --data-binary "@$WORK/config.json" \ + http://localhost/config >"$WORK/put.log" 2>&1 \ + || { cat "$WORK/put.log" >&2; die "configuration was rejected"; } + +# Poll the listener: PUT returns once the configuration is accepted, which is +# not the same instant the router has the socket bound. +i=0 +while :; do + code=$(curl -sS --max-time 5 -o "$WORK/body.txt" -w '%{http_code}' \ + "http://127.0.0.1:$PORT/index.txt" 2>>"$WORK/get.log") && break + i=$((i + 1)) + [ "$i" -lt 100 ] || { cat "$WORK/get.log" >&2; die "listener never answered"; } + sleep 0.1 +done + +[ "$code" = "200" ] || { cat "$WORK/body.txt" >&2; die "expected HTTP 200, got $code"; } +grep -q 'syscall drift canary' "$WORK/body.txt" \ + || die "served body is not the canary file" + +# Stop the daemon and let it run its teardown path *inside* the trace, so +# shutdown syscalls are part of the captured set. Signal unitd, not strace +# (see unit_pid above), then wait for strace itself to reap and flush. +UPID=$(unit_pid) || die "unitd never wrote a pid file" +kill -QUIT "$UPID" +i=0 +while kill -0 "$STRACE_PID" 2>/dev/null; do + i=$((i + 1)) + [ "$i" -lt 200 ] || die "unitd did not exit within 20s of SIGQUIT" + sleep 0.1 +done +# strace exits with the tracee's status, so this is unitd's exit code from its +# SIGQUIT teardown. Ignoring it would let a daemon that crashes on shutdown +# still produce a large, plausible capture -- and every syscall its teardown +# never reached would only be a warning. +src=0 +wait "$STRACE_PID" || src=$? +STRACE_PID= +[ "$src" -eq 0 ] || { + tail -20 "$WORK/unit.log" >&2 2>/dev/null || : + die "unitd exited $src from its SIGQUIT teardown" +} + +[ -s "$TRACE" ] || die "strace produced an empty trace" + +# Two shapes carry a syscall name: a normal entry line, and the resumption of +# a call that was interrupted by another thread's output. +# 1234 epoll_pwait(4, ...) = 1 +# 1234 <... epoll_pwait resumed>[...], 1024, ...) = 1 +# Lines for signals (--- SIGCHLD ---) and exits (+++ exited +++) match neither. +SYSCALLS=$(sed -e 's/^[0-9][0-9]*[ \t][ \t]*//' "$TRACE" \ + | sed -n \ + -e 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)(.*/\1/p' \ + -e 's/^<\.\.\. \([a-zA-Z_][a-zA-Z0-9_]*\) resumed.*/\1/p' \ + | LC_ALL=C sort -u) + +# A missing or renamed tool piped into sort -u yields an empty list, and an +# empty list compares clean against any baseline. Refuse to emit one. +COUNT=$(printf '%s\n' "$SYSCALLS" | grep -c '^[a-z]' || :) +[ "$COUNT" -ge 20 ] || die "only $COUNT syscalls extracted -- the trace did not parse" + +for required in epoll_ctl socket bind listen read write close; do + printf '%s\n' "$SYSCALLS" | grep -qx "$required" \ + || die "expected syscall '$required' is missing -- the trace did not parse" +done + +if [ -n "$OUT" ]; then + printf '%s\n' "$SYSCALLS" > "$OUT" + echo "capture.sh: wrote $COUNT syscalls to $OUT" >&2 +else + printf '%s\n' "$SYSCALLS" +fi diff --git a/test/syscalls/check.sh b/test/syscalls/check.sh new file mode 100755 index 000000000..7b3ae196d --- /dev/null +++ b/test/syscalls/check.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# +# Compare a captured syscall list (test/syscalls/capture.sh) against a +# checked-in baseline. +# +# New syscalls -> failure. Unit reached for a kernel interface it did not +# use before; that is a deliberate change and the baseline +# has to be updated in the same commit. +# Gone syscalls -> warning. Losing a syscall is not a hazard, and a runner +# or libc update legitimately drops one now and then. +# +# Usage: test/syscalls/check.sh BASELINE CAPTURED +# +# See test/syscalls/README.md for how to refresh a baseline. + +set -eu + +BASELINE=${1:?usage: $0 BASELINE CAPTURED} +CAPTURED=${2:?usage: $0 BASELINE CAPTURED} + +die() { echo "check.sh: $*" >&2; exit 1; } + +[ -f "$BASELINE" ] || die "no baseline at $BASELINE" +[ -s "$CAPTURED" ] || die "captured list $CAPTURED is missing or empty" + +# An empty or unsorted input compares clean against anything, so the two +# failure modes that would make this gate silently pass are checked first. +n_captured=$(grep -c '^[a-z]' "$CAPTURED" || :) +[ "$n_captured" -ge 20 ] \ + || die "captured list has only $n_captured entries -- refusing to compare" + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT INT TERM + +# Baselines carry '#' comments and blank lines; captures do not. +# LC_ALL=C throughout: comm rejects input sorted under a locale whose collation +# differs from the one that produced it, and '_' is exactly where they differ. +sed -e 's/#.*//' -e 's/[ \t]*$//' -e '/^$/d' "$BASELINE" | LC_ALL=C sort -u > "$work/base" +sed -e 's/#.*//' -e 's/[ \t]*$//' -e '/^$/d' "$CAPTURED" | LC_ALL=C sort -u > "$work/have" + +LC_ALL=C comm -13 "$work/base" "$work/have" > "$work/new" +LC_ALL=C comm -23 "$work/base" "$work/have" > "$work/gone" + +echo "baseline: $BASELINE ($(wc -l < "$work/base" | tr -d ' ') syscalls)" +echo "captured: $CAPTURED ($(wc -l < "$work/have" | tr -d ' ') syscalls)" + +if [ -s "$work/gone" ]; then + echo + echo "WARNING: syscalls in the baseline that were not observed:" + sed 's/^/ - /' "$work/gone" + echo " (not a failure; refresh the baseline when the drop is intended)" +fi + +if [ -s "$work/new" ]; then + echo + echo "FAIL: syscalls observed that are not in the baseline:" + sed 's/^/ + /' "$work/new" + echo + echo "If this is intended, refresh the baseline:" + echo " test/syscalls/run-in-docker.sh --update" + echo "and commit it with the change that introduced the syscall." + exit 1 +fi + +echo +echo "OK: no new syscalls." diff --git a/test/syscalls/run-in-docker.sh b/test/syscalls/run-in-docker.sh new file mode 100755 index 000000000..13a0aa2ba --- /dev/null +++ b/test/syscalls/run-in-docker.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# +# Build unitd and capture its syscall set inside a pinned container, so a +# developer's box and CI observe the same libc. +# +# Usage: +# test/syscalls/run-in-docker.sh glibc|musl [-o OUTFILE] [--update] [--check] +# +# -o OUTFILE write the captured list here +# (default: a temporary file, printed at the end) +# --check compare against the checked-in baseline (test/syscalls/check.sh) +# --update overwrite the checked-in baseline with what was captured +# +# Requires: docker. The image is pulled for linux/amd64 explicitly -- the +# baselines are per-architecture and a silent arm64 pull would produce a +# different, wrong list. + +set -eu + +die() { echo "run-in-docker.sh: $*" >&2; exit 1; } + +LIBC=${1:-} +shift 2>/dev/null || : +case "$LIBC" in + glibc|musl) ;; + *) die "usage: $0 glibc|musl [-o OUTFILE] [--update] [--check]" ;; +esac + +OUT= +UPDATE=no +CHECK=no +while [ $# -gt 0 ]; do + case "$1" in + -o) OUT=${2:?-o needs a path}; shift 2 ;; + --update) UPDATE=yes; shift ;; + --check) CHECK=yes; shift ;; + *) die "unknown argument: $1" ;; + esac +done + +REPO=$(cd "$(dirname "$0")/../.." && pwd) +BASELINE="$REPO/test/syscalls/baseline-linux-$LIBC.txt" + +# Pinned by digest, not by tag: the captured set depends on the libc, and +# debian:trixie-slim and alpine:3.22 are both mutable tags that would move a +# baseline with no change in this repository. Bumping either digest is a +# baseline refresh (README.md). +# +# The digest pins the base image, not the packages apt/apk install on top of +# it. Both are stable releases, so glibc and musl only move on a point +# release -- but that is a real residual, and one of the reasons the CI job +# is advisory rather than blocking. +if [ "$LIBC" = glibc ]; then + IMAGE=debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132 + INSTALL='apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends \ + gcc make libc6-dev libpcre2-dev strace curl ca-certificates >/dev/null' +else + IMAGE=alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce + INSTALL='apk add --no-cache gcc make musl-dev pcre2-dev strace curl >/dev/null' +fi + +command -v docker >/dev/null 2>&1 || die "docker not found" + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT INT TERM + +# With no -o the list goes to stdout, as the usage above says. Writing it to +# a temporary the EXIT trap then deletes would make the plainest invocation +# the one that throws its own result away. +TO_STDOUT=no +[ -n "$OUT" ] || { OUT="$work/captured.txt"; TO_STDOUT=yes; } + +# ptrace: Docker's default seccomp profile allows the ptrace syscall, but the +# default capability set has no CAP_SYS_PTRACE and the default AppArmor profile +# confines cross-process ptrace. strace only ever traces its own descendants +# here, so SYS_PTRACE alone is enough on a stock daemon; seccomp is left at the +# default on purpose, to keep this close to how the image really runs. +docker run --rm --platform linux/amd64 \ + --cap-add=SYS_PTRACE \ + -v "$REPO:/src:ro" \ + -w /work \ + "$IMAGE" \ + sh -eu -c " + $INSTALL + mkdir -p /work + # Copy out of the read-only mount: configure writes into the tree. + tar -C /src -cf - --exclude=./.git --exclude=./build . | tar -C /work -xf - + ./configure --prefix=/opt/unit >/dev/null + make -j\"\$(nproc)\" >/dev/null + exec test/syscalls/capture.sh + " > "$OUT.raw" 2>"$work/stderr.log" || { + cat "$work/stderr.log" >&2 + die "container run failed" + } +cat "$work/stderr.log" >&2 || : + +# The container writes progress to stderr and the list to stdout, but guard +# against a build that prints to stdout anyway: keep only syscall-shaped lines. +grep -E '^[a-z_][a-z0-9_]*$' "$OUT.raw" | LC_ALL=C sort -u > "$OUT" +rm -f "$OUT.raw" + +n=$(wc -l < "$OUT" | tr -d ' ') +[ "$n" -ge 20 ] || die "captured only $n syscalls from $IMAGE -- not trusting that" +if [ "$TO_STDOUT" = yes ]; then + echo "run-in-docker.sh: $LIBC ($IMAGE): $n syscalls" >&2 + cat "$OUT" +else + echo "run-in-docker.sh: $LIBC ($IMAGE): $n syscalls -> $OUT" >&2 +fi + +if [ "$UPDATE" = yes ]; then + { + echo "# Syscalls issued by unitd on linux/amd64, $LIBC." + echo "#" + echo "# Regenerate: test/syscalls/run-in-docker.sh $LIBC --update" + echo "# Image: $IMAGE" + echo "# See test/syscalls/README.md before editing this by hand." + cat "$OUT" + } > "$BASELINE" + echo "run-in-docker.sh: refreshed $BASELINE" >&2 +fi + +if [ "$CHECK" = yes ]; then + "$REPO/test/syscalls/check.sh" "$BASELINE" "$OUT" +fi From 85a1089318e3d40eca747bcf3ef76ddf6d57b96d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 17:23:51 +0200 Subject: [PATCH 076/391] test(docker): assert what each hardening posture actually allows Everything written down about running the FreeUnit image under a locked-down Docker was either folklore or wrong. A 2026-07 audit had it that Docker's default seccomp profile blocks capget and mount; measured against docker 29.7.2 it blocks neither -- capget is allowed by name in moby's default profile, and mount is gated on CAP_SYS_ADMIN plus the default AppArmor profile. Meanwhile the questions people actually ask -- can I run this non-root, can I drop every capability, does no-new-privileges break it -- had no answer in the repository at all. Answer them with a script that starts the image once per posture with the same trivial Python application and asserts the outcome. Six cells, all under Docker's default seccomp profile, unchanged: root, default caps 200 root, --security-opt no-new-privileges 200 root, --cap-drop=ALL refused --user 1000:1000 + tmpfs on state/run/tmp 200 --user 1000:1000 --cap-drop=ALL + tmpfs 200 --user 1000:1000 --cap-drop=ALL --no-new-privs + tmpfs 200 Four of those are worth knowing. --cap-drop=ALL and root do not mix: the daemon's first act is dropping the router to the unprivileged unit user, and setgid(999) then fails with EPERM -- capabilities are worth dropping after the process is non-root, not instead of it. no-new-privileges is free, since setuid from root is not a privilege gain. A bare --tmpfs /var/lib/unit is not enough, because Docker creates a tmpfs root-owned 0755 and a non-root container still gets EACCES; the ownership has to be spelled out. And --user 1000 is not --user 1000:1000: uid 1000 has no passwd entry in this image, so a bare uid resolves its primary group to GID 0 -- $ docker run --rm --user 1000 --entrypoint sh freeunit:test -c id uid=1000 gid=0(root) groups=0(root) -- and the container keeps root-group access, which is not what anyone means by "runs unprivileged". Those last two are the steps people miss, and now they are tests rather than sentences someone may or may not have written down. Every cell asserts a cause, not just an outcome. The refusal cell greps the container log for setgid(999), because "no 200" on its own is satisfied by a wholly broken image. The five serving cells check two things. What the canary reports about itself -- uid, gid and os.getgroups(), because dropping the primary gid while keeping group 0 is root-group access under another name -- and the router's uid and gid, read from /proc inside the container, because a router still running as root would serve a correctly-dropped application perfectly well. Inside, not `docker top`: that reports host-side uids, which under daemon user-namespace remapping are the remapped ones and would fail every cell. The scan matches the start of cmdline and skips its own pid -- the probing shell's command line contains the string it searches for, and in a non-root cell it runs as 1000, so a substring match would have reported the right answer whatever the router did. That one was caught by negative-testing the assertion rather than by reading it. Root cells expect 999 throughout, which is the image's unit user, so the drop is measured rather than assumed; non-root cells expect 1000. The application half has to come from the application: `docker exec ... id` reports the container's configured user, a different process, and would look right even if Unit dropped to the wrong credentials. The three non-root cells share one flag variable rather than repeating it, after the flags and the labels describing them drifted apart while this was being written: two cells said --user 1000:1000 and passed --user 1000, which is uid 1000 with GID 0 and not what either label meant. The wait before probing is two-phase. docker-entrypoint.sh runs a temporary daemon to apply the initial configuration -- which activates the listener -- and only then execs the real one, so polling immediately can take a 200 from the throwaway daemon and pass while the production startup is broken. Each cell waits for the entrypoint's "ready for start up" line and refuses to probe without it, since a container that stalls after configuration is exactly that failure with the temporary daemon still answering. Both waits are absolute deadlines rather than iteration counts -- with a per-request timeout, an iteration bound bounds attempts and not time, and a listener that accepts without answering would spend a hundred five-second timeouts in one cell -- and every curl has its own deadline on top. The identity patterns are anchored to end of line, because an unanchored gid=999 also matches gid=9990. A refused cell is not pinned to one shade of failure -- never started, never ready, never answered -- because which one a broken posture produces is not stable; --cap-drop=ALL leaves the entrypoint's daemon alive but never ready. The log assertion is what names the cause. The workflow's path filters gain the files that decide the credentials the canary reports: src/nxt_capability.*, src/nxt_credential.*, src/nxt_isolation.*, src/nxt_process.*, src/nxt_runtime.*, src/nxt_main_process.* and src/nxt_application.*. Its timeout goes from 15 to 20 minutes; the matrix measured 57 seconds locally, but the image build dominates that job and there is no reason to run it close to the edge. It runs as the last step of the existing seccomp workflow, against the image that workflow already builds from the in-repo Dockerfile: the expensive part of a Docker check is the image build, and this is the second security property of the same artifact. That workflow's path filters gain src/nxt_capability.*, src/nxt_credential.*, src/nxt_isolation.*, src/nxt_process.* and src/nxt_runtime.* -- the table above is a statement about what those files do (nxt_runtime_conf_init() is where rt->user_cred and rt->group are set and nxt_capability_set() is called), so a change to them has to re-run it. The workflow is still named after AF_ALG, though: renaming it would rename the check, so that is left to a change of its own. The platform is pinned to linux/amd64, which is what CI runs and stops a multi-arch tag resolving to something else by ambient default; UNIT_HARDENING_PLATFORM overrides it on an arm64 host, where the arm64 manifest is the artifact worth testing. Co-Authored-By: Claude Opus 5 --- .github/workflows/security-seccomp.yml | 45 +++- test/security/docker/README.md | 92 ++++++++ test/security/docker/app/config.json | 15 ++ test/security/docker/app/wsgi.py | 24 ++ test/security/docker/test-hardening.sh | 315 +++++++++++++++++++++++++ test/syscalls/capture.sh | 6 +- test/syscalls/check.sh | 6 +- test/syscalls/run-in-docker.sh | 6 +- 8 files changed, 505 insertions(+), 4 deletions(-) create mode 100644 test/security/docker/README.md create mode 100644 test/security/docker/app/config.json create mode 100644 test/security/docker/app/wsgi.py create mode 100755 test/security/docker/test-hardening.sh diff --git a/.github/workflows/security-seccomp.yml b/.github/workflows/security-seccomp.yml index f9dc377bd..7db916cc5 100644 --- a/.github/workflows/security-seccomp.yml +++ b/.github/workflows/security-seccomp.yml @@ -5,13 +5,45 @@ on: paths: - 'pkg/docker/seccomp-no-af-alg.json' - 'pkg/docker/Dockerfile.*' + - 'pkg/docker/docker-entrypoint.sh' - 'test/security/seccomp/**' + - 'test/security/docker/**' + # The hardening matrix documents what the daemon does with credentials + # and capabilities; these are the files that decide it. + - 'src/nxt_capability.*' + - 'src/nxt_credential.*' + - 'src/nxt_isolation.*' + - 'src/nxt_process.*' + # nxt_runtime_conf_init() sets rt->user_cred/rt->group and calls + # nxt_capability_set(), which is the decision this table describes. + - 'src/nxt_runtime.*' + # nxt_main_process.c:556 fills app_conf->user from rt->user_cred, and + # nxt_application.c:669 sets process->user_cred -- the credentials the + # canary reports back. + - 'src/nxt_main_process.*' + - 'src/nxt_application.*' - '.github/workflows/security-seccomp.yml' pull_request: paths: - 'pkg/docker/seccomp-no-af-alg.json' - 'pkg/docker/Dockerfile.*' + - 'pkg/docker/docker-entrypoint.sh' - 'test/security/seccomp/**' + - 'test/security/docker/**' + # The hardening matrix documents what the daemon does with credentials + # and capabilities; these are the files that decide it. + - 'src/nxt_capability.*' + - 'src/nxt_credential.*' + - 'src/nxt_isolation.*' + - 'src/nxt_process.*' + # nxt_runtime_conf_init() sets rt->user_cred/rt->group and calls + # nxt_capability_set(), which is the decision this table describes. + - 'src/nxt_runtime.*' + # nxt_main_process.c:556 fills app_conf->user from rt->user_cred, and + # nxt_application.c:669 sets process->user_cred -- the credentials the + # canary reports back. + - 'src/nxt_main_process.*' + - 'src/nxt_application.*' - '.github/workflows/security-seccomp.yml' workflow_dispatch: @@ -30,7 +62,9 @@ jobs: seccomp-af-alg: name: Verify AF_ALG seccomp mitigation runs-on: ubuntu-latest - timeout-minutes: 15 + # Raised from 15 with the hardening matrix added at the end. The image + # build dominates; the matrix itself measured 57 seconds locally. + timeout-minutes: 20 steps: - uses: actions/checkout@v5 @@ -79,3 +113,12 @@ jobs: - name: Run seccomp tests (FreeUnit image) run: bash test/security/seccomp/test-af-alg.sh freeunit:seccomp-test + + # Reuses the image built above rather than building or pulling another + # one: this is the second security property of the same artifact, and an + # image build is the expensive part of this workflow. + # + # (The workflow is still named after AF_ALG. Renaming it would rename + # the check, so that is left to a deliberate change of its own.) + - name: Run Docker hardening matrix (FreeUnit image) + run: bash test/security/docker/test-hardening.sh freeunit:seccomp-test diff --git a/test/security/docker/README.md b/test/security/docker/README.md new file mode 100644 index 000000000..189cb1f1c --- /dev/null +++ b/test/security/docker/README.md @@ -0,0 +1,92 @@ +# Docker hardening matrix + +`test-hardening.sh` starts the FreeUnit image once per hardening posture with +the same trivial Python application and asserts the outcome. It is +documentation that runs: the point is not that any of these postures is +required, but that the file cannot go stale about which ones work. + +```sh +./test/security/docker/test-hardening.sh # published image +./test/security/docker/test-hardening.sh freeunit:local # a locally built one +``` + +CI runs it as the last step of `.github/workflows/security-seccomp.yml`, +against the image that workflow has already built from the in-repo Dockerfile. +That workflow also triggers on `src/nxt_capability.*`, `src/nxt_credential.*`, +`src/nxt_isolation.*`, `src/nxt_process.*` and `src/nxt_runtime.*`, since +those are the files that decide what the table below says. + +The platform is pinned to `linux/amd64` -- what CI runs, and so that a +multi-arch tag cannot resolve to something else by ambient default. On an +arm64 host set `UNIT_HARDENING_PLATFORM=linux/arm64`. + +## What it establishes + +| posture | outcome | +|---|---| +| root, default capabilities | serves 200 | +| root, `--security-opt no-new-privileges` | serves 200 | +| root, `--cap-drop=ALL` | refused, never reaches readiness: `setgid(999) failed (1: Operation not permitted)` | +| `--user 1000:1000` + writable tmpfs on state/run/tmp | serves 200 | +| `--user 1000:1000 --cap-drop=ALL` + tmpfs | serves 200 | +| `--user 1000:1000 --cap-drop=ALL --security-opt no-new-privileges` + tmpfs | serves 200 | + +Every cell runs under Docker's **default seccomp profile**, unchanged. + +Cells assert a cause, not just an outcome. The refusal cell greps the +container log for `setgid(999) failed`, because a wholly broken image would +satisfy "no 200" too. The five serving cells check two things, both anchored to end of line so that +`gid=999` cannot be satisfied by `gid=9990`: + +- what the canary reports about **itself** -- `uid`, `gid` *and* + `os.getgroups()`, since a process that drops its primary gid while keeping + group 0 has root-group access under another name. It has to come from the + application: `docker exec ... id` reports the container's configured user, + a different process, and would look right even if Unit dropped the + application to the wrong credentials. +- the **router**'s uid and gid, read from `/proc` *inside* the container, + because a router still running as root would serve a correctly-dropped + application perfectly well. Not `docker top`: that reports host-side uids, + which under daemon user-namespace remapping are the remapped ones and would + turn every cell red. The scan matches the *start* of `cmdline` and skips + its own pid -- the probing shell's command line contains the string it is + searching for, and in a non-root cell it runs as 1000, so a substring match + would have reported the right answer whatever the router did. + +Root cells expect `uid=999 gid=999 groups=999` and a router at 999 -- the +image's `unit` user, so the drop is measured rather than assumed. Non-root +cells expect 1000 throughout. A 200 on its own says nothing about who served +it, and the flag list and the label describing it did in fact drift apart +once while this was being written. + +The wait is two-phase for the same reason. `docker-entrypoint.sh` runs a +temporary daemon to apply the initial configuration, which activates the +listener, and only then execs the real one; polling immediately can take a +200 from the throwaway and pass while production startup is broken. Each +cell waits for the entrypoint's "ready for start up" line first. + +Three things worth taking away: + +- **`--cap-drop=ALL` and root do not mix.** The daemon's first act is to drop + the router and applications to the unprivileged `unit` user, and that needs + CAP_SETGID/CAP_SETUID. Dropping capabilities is worth doing *after* the + process is already non-root, not instead of it. +- **`no-new-privileges` is free.** `setuid` from root is not a privilege gain, + so the bit never applies to what Unit does; it costs nothing in either + posture. +- **A bare `--tmpfs /var/lib/unit` is not enough.** Docker creates a tmpfs + root-owned and 0755, so a non-root container still gets EACCES on the state, + run and tmp directories. The ownership has to be spelled out: + `--tmpfs /var/lib/unit:uid=1000,gid=1000`. This is the step people miss. +- **`--user 1000` is not `--user 1000:1000`.** Uid 1000 has no passwd entry + in this image, so a bare uid resolves its primary group to GID 0 and the + container keeps root-group access. Spell out the group. + +## What it does not establish + +It does not test the shipped `pkg/docker/seccomp-no-af-alg.json` profile -- +that has its own test in `test/security/seccomp/` -- and it makes no claim +that Docker's default seccomp profile blocks `capget` or `mount`. Measured +against docker 29.7.2 it blocks neither: `capget` is allowed by name in +moby's default profile, and `mount` is gated on CAP_SYS_ADMIN plus the default +AppArmor profile, not on seccomp. Documentation that says otherwise is wrong. diff --git a/test/security/docker/app/config.json b/test/security/docker/app/config.json new file mode 100644 index 000000000..aff0109bb --- /dev/null +++ b/test/security/docker/app/config.json @@ -0,0 +1,15 @@ +{ + "listeners": { + "*:8080": { + "pass": "applications/canary" + } + }, + "applications": { + "canary": { + "type": "python", + "path": "/www/", + "module": "wsgi", + "processes": 1 + } + } +} diff --git a/test/security/docker/app/wsgi.py b/test/security/docker/app/wsgi.py new file mode 100644 index 000000000..3578792a1 --- /dev/null +++ b/test/security/docker/app/wsgi.py @@ -0,0 +1,24 @@ +"""Smallest application that proves the whole request path works. + +It reports the credentials of the process that served the request. That is +the only way the hardening matrix can check *who* answered: `docker exec ... +id` would report the container's configured user, which is a different +process and would look right even if Unit dropped to the wrong credentials. + +Supplementary groups are part of that: a process can drop its primary gid and +still hold group 0 if setgroups() was skipped, which is root-group access by +another name. +""" + +import os + + +def application(environ, start_response): + body = ("docker hardening canary uid=%d gid=%d groups=%s\n" + % (os.getuid(), os.getgid(), + ",".join(str(g) for g in sorted(os.getgroups())))).encode() + start_response( + "200 OK", + [("Content-Type", "text/plain"), ("Content-Length", str(len(body)))], + ) + return [body] diff --git a/test/security/docker/test-hardening.sh b/test/security/docker/test-hardening.sh new file mode 100755 index 000000000..f07f573d5 --- /dev/null +++ b/test/security/docker/test-hardening.sh @@ -0,0 +1,315 @@ +#!/bin/sh +# +# What the FreeUnit image can and cannot do under each common Docker +# hardening posture. Every cell starts the same image with the same trivial +# Python application and asserts the documented outcome -- a served 200, or a +# refusal at a named point. +# +# This is documentation that runs. It is not a claim that any of these +# postures is required, and it deliberately does not claim that Docker's +# default seccomp profile blocks capget or mount: measured against docker +# 29.7.2 it blocks neither (capget is allowed by name; mount is gated on +# CAP_SYS_ADMIN and AppArmor). Every cell below therefore runs with the +# default seccomp profile, unchanged; the profile FreeUnit does ship +# (pkg/docker/seccomp-no-af-alg.json) has its own test next door in +# test/security/seccomp/. +# +# Usage: +# ./test/security/docker/test-hardening.sh [IMAGE] +# +# IMAGE defaults to the published python-slim flavour. Requires docker and +# curl on the host. + +set -eu + +IMAGE="${1:-ghcr.io/freeunitorg/freeunit:latest-python-3.13-slim}" +HERE=$(cd "$(dirname "$0")" && pwd) +APP="$HERE/app" +PORT="${UNIT_HARDENING_PORT:-8099}" +# Pinned rather than left to the daemon's ambient default, so a multi-arch tag +# cannot silently resolve to something other than what CI runs. Override it +# on an arm64 host, where the arm64 manifest is the artifact worth testing: +# UNIT_HARDENING_PLATFORM=linux/arm64 ./test-hardening.sh +PLATFORM="${UNIT_HARDENING_PLATFORM:-linux/amd64}" + +PASS=0 +FAIL=0 +NAME= +RESULT= + +die() { echo "test-hardening.sh: $*" >&2; exit 1; } + +command -v docker >/dev/null 2>&1 || die "docker not found" +command -v curl >/dev/null 2>&1 || die "curl not found" + +LOGDIR=$(mktemp -d) + +cleanup() { + [ -n "$NAME" ] && { docker rm -f "$NAME" >/dev/null 2>&1 || :; } + rm -rf "$LOGDIR" + return 0 +} + +# A trapped signal does not end a POSIX shell: without an explicit exit the +# script would resume, with LOGDIR already deleted underneath it, and report +# an ordinary test error instead of stopping. The EXIT trap is cleared first +# so cleanup runs once. +trap cleanup EXIT +trap 'trap - EXIT; cleanup; exit 130' INT +trap 'trap - EXIT; cleanup; exit 143' TERM + +# Start the image with the given docker flags, wait for the application, and +# set RESULT to "200", "no-200" or "no-start". It sets a variable rather than +# echoing, because a $(command substitution) would run it in a subshell: NAME +# would never reach the parent, and the EXIT/INT/TERM trap could not remove a +# container left behind by a Ctrl-C -- leaking both the container and the +# published port. Never fails the script itself: a cell that is *expected* +# to be refused has to be able to report the refusal. +attempt() { + NAME="unit-hardening-$$" + docker rm -f "$NAME" >/dev/null 2>&1 || : + + # No --rm: a container that exits on its own would delete itself along + # with the logs, and the logs are the whole point of a refused cell. + # shellcheck disable=SC2086 # $1 is a deliberate list of docker flags + docker run -d --name "$NAME" --platform "$PLATFORM" \ + -v "$APP:/www:ro" \ + -v "$APP/config.json:/docker-entrypoint.d/config.json:ro" \ + -p "127.0.0.1:$PORT:8080" \ + $1 "$IMAGE" >/dev/null 2>"$LOGDIR/start.err" || { + cp "$LOGDIR/start.err" "$LOGDIR/container.log" + # docker run can fail *after* creating the container -- a port + # already allocated, for one -- leaving it in Created state. The + # trap cannot reach it once NAME is cleared, and the next run + # picks a different pid-based name, so remove it here. + docker rm -f "$NAME" >/dev/null 2>&1 || : + NAME= + RESULT=no-start + return 0 + } + + # docker-entrypoint.sh runs a *temporary* daemon to apply the initial + # configuration -- which activates this listener -- and only then execs + # the real one. Polling straight away can therefore get a 200 out of the + # throwaway daemon and pass while the production startup is broken. Wait + # for the entrypoint to say it is done first. + # Absolute deadlines, not iteration counts: with a per-request timeout an + # iteration bound is a bound on attempts, not on time, and a listener that + # accepts and never answers would spend a hundred five-second timeouts + # here -- eight minutes in one cell, against a twenty-minute job. + ready=no + deadline=$(( $(date +%s) + 45 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + if docker logs "$NAME" 2>&1 | grep -q 'ready for start up'; then + ready=yes + break + fi + docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null \ + | grep -q true || break + sleep 0.2 + done + + # Never probe an unready container. Exhausting the loop while it is still + # running means the entrypoint stalled after applying the configuration -- + # with the temporary daemon still answering, so a probe would take a 200 + # from it and pass the very failure this wait exists to catch. + if [ "$ready" = no ]; then + docker logs "$NAME" > "$LOGDIR/container.log" 2>&1 || : + running=$(docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null || echo false) + docker rm -f "$NAME" >/dev/null 2>&1 || : + NAME= + if [ "$running" = true ]; then + RESULT=no-ready + else + RESULT=no-200 + fi + return 0 + fi + + deadline=$(( $(date +%s) + 30 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + # --max-time as well as the deadline: without it one wedged request + # blocks past the deadline the loop can only check between attempts. + code=$(curl -s --max-time 5 -o "$LOGDIR/body.txt" -w '%{http_code}' \ + "http://127.0.0.1:$PORT/" 2>/dev/null || echo 000) + if [ "$code" = "200" ] && grep -q 'docker hardening canary' "$LOGDIR/body.txt"; then + docker logs "$NAME" > "$LOGDIR/container.log" 2>&1 || : + # The canary reports the credentials of the process that served + # the request, so a cell can assert who answered rather than + # trusting the flags in its own label. + sed 's/^/response: /' "$LOGDIR/body.txt" >> "$LOGDIR/container.log" + # ...and the router's own credentials, which the canary cannot + # speak for: a router still running as root would serve a + # correctly-dropped application just fine. Read from /proc inside + # the container rather than `docker top`, which reports host-side + # uids -- under daemon user-namespace remapping those are the + # remapped ones and would fail every cell. The image has no ps. + # + # Match on the *start* of cmdline, not a substring of it: this + # shell's own cmdline contains the pattern it is searching for, + # and in a non-root cell it runs as 1000 -- so a substring match + # would report the right answer no matter what the router did. + docker exec "$NAME" sh -c ' + for d in /proc/[0-9]*; do + [ "$d" = "/proc/$$" ] && continue + c=$(tr "\0" " " < "$d/cmdline" 2>/dev/null) || continue + case "$c" in "unit: router"*) ;; *) continue ;; esac + awk "/^Uid:/ {u=\$2} /^Gid:/ {g=\$2} + END {print \"router uid=\" u \" gid=\" g}" \ + "$d/status" + done' 2>/dev/null \ + | sed 's/^/proc: /' >> "$LOGDIR/container.log" || : + docker rm -f "$NAME" >/dev/null 2>&1 || : + NAME= + RESULT=200 + return 0 + fi + # A container that has already exited will never answer. + docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null \ + | grep -q true || break + sleep 0.2 + done + + docker logs "$NAME" > "$LOGDIR/container.log" 2>&1 || : + docker rm -f "$NAME" >/dev/null 2>&1 || : + NAME= + RESULT=no-200 +} + +# cell DESCRIPTION 200|refused FLAGS [LOG-SUBSTRING] +# +# LOG-SUBSTRING is how a cell is pinned to its cause. "no-200" on its own is +# satisfied by a wholly broken image, so a cell documenting a refusal has to +# say which refusal; and a 200 says nothing about who served it, so every +# serving cell matches on the credentials the canary reports. Both the +# container log and the response body are searched. +cell() { + desc=$1 + expect=$2 + flags=$3 + want_log=${4:-} + + RESULT= + attempt "$flags" + got=$RESULT + + # "refused" is any of the ways a cell can fail to serve: the container + # never started, never reached readiness, or never answered. Which one a + # broken posture produces is not stable -- --cap-drop=ALL leaves the + # entrypoint's daemon alive but never ready -- and the cause is pinned by + # the log assertion below, not by which shade of failure it was. + case "$expect" in + refused) + case "$got" in + no-start|no-ready|no-200) got=refused ;; + esac + ;; + esac + + if [ "$got" != "$expect" ]; then + echo "FAIL: $desc -> $got, expected $expect" + [ -f "$LOGDIR/container.log" ] && sed 's/^/ | /' "$LOGDIR/container.log" + FAIL=$((FAIL + 1)) + return 0 + fi + + # want_log holds one pattern per line and every one has to match. + if [ -n "$want_log" ]; then + oldifs=$IFS + IFS=' +' + for pat in $want_log; do + IFS=$oldifs + if ! grep -q "$pat" "$LOGDIR/container.log"; then + echo "FAIL: $desc -> $got as expected, but the container log" \ + "does not match \"$pat\"" + [ -f "$LOGDIR/container.log" ] \ + && sed 's/^/ | /' "$LOGDIR/container.log" + FAIL=$((FAIL + 1)) + return 0 + fi + IFS=' +' + done + IFS=$oldifs + fi + + echo "PASS: $desc -> $got (as documented)" + PASS=$((PASS + 1)) +} + +echo "Image: $IMAGE" +echo "Platform: $PLATFORM" +echo "Every cell runs with Docker's DEFAULT seccomp profile." +echo "" + +# Started as root, the daemon drops the router and applications to the image's +# unprivileged "unit" user (uid 999). The canary reports what it actually ran +# as, so these cells assert the drop happened rather than assuming it. +# Two patterns, both anchored -- an unanchored "gid=999" also matches gid=9990. +# The first is the application's own view of itself, including supplementary +# groups: dropping the primary gid while keeping group 0 is root-group access +# under another name. The second is the router, which the application cannot +# speak for. +ROOT_ID="uid=999 gid=999 groups=999$ +^proc: router uid=999 gid=999$" + +# The shipped default: root in the container, full default capability set. +cell "root, default caps" 200 "" "$ROOT_ID" + +# no-new-privileges does not stop Unit dropping privileges: setuid/setgid from +# root is not a privilege *gain*, so the bit never applies to it. +cell "root, --security-opt no-new-privileges" 200 \ + "--security-opt no-new-privileges" "$ROOT_ID" + +# Root without capabilities cannot become the unprivileged 'unit' user, which +# is the first thing the daemon does: +# [alert] setgid(999) failed (1: Operation not permitted) +# CAP_SETGID and CAP_SETUID are what --cap-drop=ALL takes away here. Dropping +# capabilities is only useful once the process is already non-root, which is +# the cell below. Documented as a refusal, not a bug. +cell "root, --cap-drop=ALL" refused "--cap-drop=ALL" \ + "setgid(999) failed" + +# The non-root recipe. 1000:1000, not a bare 1000: uid 1000 has no passwd +# entry in this image, so `--user 1000` resolves the primary group to GID 0 -- +# +# $ docker run --rm --user 1000 --entrypoint sh IMAGE -c id +# uid=1000 gid=0(root) groups=0(root) +# +# -- and the container keeps root-group access, which is not what anyone means +# by "runs unprivileged". +# +# Nothing to drop to means nothing needs CAP_SETUID -- but the state, run and +# tmp directories are baked into the image root-owned, so they have to be +# replaced by writable mounts. A bare "--tmpfs /var/lib/unit" is not enough: +# a tmpfs is created root-owned 0755 and the daemon still gets EACCES, so the +# ownership has to be spelled out. This is the part of the non-root recipe +# people get wrong. +# +# One variable for all three cells, so a flag list can never drift from the +# label describing it -- which is exactly what happened while writing this. +# Each cell also asserts "uid=1000 gid=1000" from the canary's own +# os.getuid()/os.getgid(), so the identity of the process that served the +# request is measured rather than assumed. +NONROOT="--user 1000:1000" +NONROOT="$NONROOT --tmpfs /var/lib/unit:uid=1000,gid=1000" +NONROOT="$NONROOT --tmpfs /var/run:uid=1000,gid=1000" +NONROOT="$NONROOT --tmpfs /var/tmp:mode=1777" +NONROOT_ID="uid=1000 gid=1000 groups=1000$ +^proc: router uid=1000 gid=1000$" + +cell "--user 1000:1000, tmpfs state/run/tmp" 200 \ + "$NONROOT" "$NONROOT_ID" + +# The same, with every capability removed. Nothing above needed one. +cell "--user 1000:1000, --cap-drop=ALL, tmpfs" 200 \ + "$NONROOT --cap-drop=ALL" "$NONROOT_ID" + +# And with no-new-privileges on top: the posture to recommend. +cell "--user 1000:1000, --cap-drop=ALL, no-new-privileges, tmpfs" 200 \ + "$NONROOT --cap-drop=ALL --security-opt no-new-privileges" "$NONROOT_ID" + +echo "" +echo "Results: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] diff --git a/test/syscalls/capture.sh b/test/syscalls/capture.sh index 3f5d11a0a..1fbaed9d7 100755 --- a/test/syscalls/capture.sh +++ b/test/syscalls/capture.sh @@ -69,7 +69,11 @@ cleanup() { rm -rf "$WORK" return 0 } -trap cleanup EXIT INT TERM +# A trapped signal does not end a POSIX shell, so each signal handler exits +# explicitly; the EXIT trap is cleared first so the cleanup runs once. +trap cleanup EXIT +trap 'trap - EXIT; cleanup; exit 130' INT +trap 'trap - EXIT; cleanup; exit 143' TERM mkdir -p "$WORK/state" "$WORK/tmp" "$WORK/modules" "$WORK/share" echo 'syscall drift canary' > "$WORK/share/index.txt" diff --git a/test/syscalls/check.sh b/test/syscalls/check.sh index 7b3ae196d..3b89ed4f7 100755 --- a/test/syscalls/check.sh +++ b/test/syscalls/check.sh @@ -30,7 +30,11 @@ n_captured=$(grep -c '^[a-z]' "$CAPTURED" || :) || die "captured list has only $n_captured entries -- refusing to compare" work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT INT TERM +# A trapped signal does not end a POSIX shell, so each signal handler exits +# explicitly; the EXIT trap is cleared first so the cleanup runs once. +trap 'rm -rf "$work"' EXIT +trap 'trap - EXIT; rm -rf "$work"; exit 130' INT +trap 'trap - EXIT; rm -rf "$work"; exit 143' TERM # Baselines carry '#' comments and blank lines; captures do not. # LC_ALL=C throughout: comm rejects input sorted under a locale whose collation diff --git a/test/syscalls/run-in-docker.sh b/test/syscalls/run-in-docker.sh index 13a0aa2ba..235c83e73 100755 --- a/test/syscalls/run-in-docker.sh +++ b/test/syscalls/run-in-docker.sh @@ -63,7 +63,11 @@ fi command -v docker >/dev/null 2>&1 || die "docker not found" work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT INT TERM +# A trapped signal does not end a POSIX shell, so each signal handler exits +# explicitly; the EXIT trap is cleared first so the cleanup runs once. +trap 'rm -rf "$work"' EXIT +trap 'trap - EXIT; rm -rf "$work"; exit 130' INT +trap 'trap - EXIT; rm -rf "$work"; exit 143' TERM # With no -o the list goes to stdout, as the usage above says. Writing it to # a temporary the EXIT trap then deletes would make the plainest invocation From c244773e4a4f14b4e622b677d8898d273357a494 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 30 Aug 2026 22:07:11 +0200 Subject: [PATCH 077/391] test(router): drain the soak test's destination port on teardown nxt_router_start_fail_soak_test() released dport with a bare nxt_port_use(task, dport, -1), but the successful start at the end of the test is queued rather than written, and nxt_port_msg_chk_insert() takes a port reference for every queued message. The port therefore stayed above zero and never reached nxt_port_mp_cleanup(), whose assertions -- descriptors closed, use_count zero, message queue and rpc hashes empty -- are the ones a --debug build exists to run. So the soak's "green in --debug" was weaker than it looked: it was green partly because the port leaked, not because the fixture was clean. The port's mem_pool and its queued nxt_port_send_msg_t leaked with it, and the message's buf pointed into a pool the test destroys moments later. Confirmed under gdb before the change: after the soak test's teardown only port id=0 reached nxt_port_mp_cleanup(), while the sibling single-shot test tore down both of its ports. After the change the soak tears down both, each at use_count 0. Drain with nxt_port_test_run_error_handler(), the existing NXT_TESTS wrapper that src/test/nxt_port_fail_test.c already uses for this, and then require the queue to be empty rather than assume it. That check is not decoration: with the drain removed and the check kept, the suite exits 1 reporting the port still holds queued messages. The buffer completion handlers the drain queues onto the fixture's work queue never run, which is safe rather than lossy -- nxt_buf_completion() would only return the buffers to the same pool nxt_mp_destroy() frees wholesale, and the queued work items live in cache chunks that nxt_work_queue_cache_destroy() frees the same way. Found while reviewing #255, whose new test had the same shape and fixes it the same way. Closes #258 Co-Authored-By: Claude Opus 5 --- src/test/nxt_router_start_fail_soak_test.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/test/nxt_router_start_fail_soak_test.c b/src/test/nxt_router_start_fail_soak_test.c index 57c83231f..8ba7d88d9 100644 --- a/src/test/nxt_router_start_fail_soak_test.c +++ b/src/test/nxt_router_start_fail_soak_test.c @@ -308,6 +308,23 @@ nxt_router_start_fail_soak_test(nxt_thread_t *thr) done: if (dport != NULL) { + /* + * The successful start above was queued rather than written, and + * nxt_port_msg_chk_insert() takes a port reference for every queued + * message. Releasing only the test's own reference leaves the port + * above zero, so it never reaches nxt_port_mp_cleanup() and the + * assertions there -- the ones a --debug build exists to run -- + * never execute for it. + */ + nxt_port_test_run_error_handler(task, dport); + + if (!nxt_queue_is_empty(&dport->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start fail soak test: destination port " + "still holds queued messages after teardown"); + ret = NXT_ERROR; + } + nxt_port_use(task, dport, -1); } From 883811a99613c97188883afe0761166f7fa2b59b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:54:32 +0000 Subject: [PATCH 078/391] build(deps): bump softprops/action-gh-release from 3.0.2 to 3.0.3 Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/3d0d9888cb7fd7b750713d6e236d1fcb99157228...efb35369e0ad2afab669f228072c1b0d510eae64) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-version: 3.0.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build-deb.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 32a940176..d60a5aa24 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -452,7 +452,7 @@ jobs: NOTES - name: Attach .deb to GitHub Release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: body_path: RELEASE_NOTES.md files: | From 3e3ce864c73bc3d1d5500dab381f84d23f6bea73 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 3 Sep 2026 03:38:47 +0200 Subject: [PATCH 079/391] main: reply to every START_PROCESS that cannot start a process START_PROCESS carries the stream of an RPC the router has already armed in nxt_router_start_app_process_handler(), and only a reply retires it. Five exits from nxt_main_start_process_handler() returned without one. Four are "goto close_fds" taken before any reply exists: the router port is not in the runtime, the sender's SCM_CREDENTIALS pid is not the router's, nxt_process_new() returns NULL, or the process mem_pool cannot be created. The fifth is at "failed:" itself, where the RPC_ERROR write is guarded by "if (port != NULL)" with no else, so a reply port that cannot be found drops the answer silently. The router never learns. Neither nxt_router_app_port_ready() nor nxt_router_app_port_error() runs, so app->proto_port_requests keeps the count incremented at nxt_router.c:434 and :482; every later start for that application parks on a prototype that is not coming, and the requests waiting on it are never failed. The router's app_joint_rpc registers no peer (nxt_router.c:477-480), so neither nxt_port_rpc_remove_peer() nor nxt_port_rpc_close() can recover the stream once main has returned in silence; nothing retries. Only the exits above the fork() are affected. A process that dies after it -- before or after PROCESS_CREATED -- is already answered for: main's SIGCHLD reaper zeroes process->stream only for a process that reached READY (nxt_main_process.c:1108-1110) and then notifies unconditionally, so a death while CREATING sends REMOVE_PID still carrying the router's stream (nxt_port.c:854), and nxt_router_remove_pid_handler() rewrites that into RPC_ERROR (nxt_router.c:1128-1133). PR #255 cleared the counter when a prototype start fails; this is the remaining case, where main exits before there is a start to fail. Route the four into "failed:", which already writes RPC_ERROR: initialise process to NULL, guard the nxt_process_use() there, and drop the now unreachable close_fds label, so that a future early return can only escape by stepping over the label that would have answered for it. Give the port lookup an else that says so in the log; a dead sender's RPCs die with it, but a silent drop is the shape of the defect above. Key the reply-port lookup on nxt_recv_msg_cmsg_pid(msg) instead of msg->port_msg.pid. The first two paths now reach the reply before the router-identity check has run, and port_msg.pid is filled in by the sender, so answering the pid a message claims would let any process that can write to main's port name the router as reply target and cancel a stream of the router's choosing. On the credential it can only ever retire an RPC of the sender's own. This changes nothing for a legitimate sender: nxt_port_socket.c:213 is the only production site that sets port_msg.pid and it sets nxt_pid, every pre-existing reply path sits after the identity check that already requires the two to be equal, and where SCM_CREDENTIALS is unavailable nxt_recv_msg_cmsg_pid() is defined as port_msg.pid (nxt_port.h:242). A prototype start begun by prefork arms nxt_router_app_prefork_error() (nxt_router.c:3431) rather than the app-port path, and it too has been waiting on a reply that never came -- a configuration apply that hit one of these exits hung instead of failing. The test drives the handler directly with a synthesised runtime and message: a reply must land on the reply port, carrying the incoming stream and marked last, for the router-port, credential-mismatch and process-allocation exits, and nothing may be queued anywhere when the sender owns no port at all. It goes red on the unfixed handler ("left 0 replies on the reply port ... the START_PROCESS RPC is stranded") and red again if only the RPC_ERROR write is suppressed. Closes #257. Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + src/nxt_main_process.c | 116 ++++- src/nxt_main_process.h | 6 + src/test/nxt_main_start_process_reply_test.c | 508 +++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 6 files changed, 624 insertions(+), 12 deletions(-) create mode 100644 src/test/nxt_main_start_process_reply_test.c diff --git a/auto/sources b/auto/sources index 0f97d65e4..529c0ba77 100644 --- a/auto/sources +++ b/auto/sources @@ -183,6 +183,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_start_fail_soak_test.c \ src/test/nxt_router_proto_wedge_test.c \ src/test/nxt_router_remove_pid_soak_test.c \ + src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_fd_test.c \ " diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index fdb52aa05..027211977 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -67,6 +67,42 @@ static nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *tmp_name, static void nxt_main_port_access_log_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +#if (NXT_TESTS) +static nxt_uint_t nxt_main_test_process_new_failure_count; + + +void +nxt_main_test_process_new_failures(nxt_uint_t failures) +{ + nxt_main_test_process_new_failure_count = failures; +} + + +/* + * nxt_process_new() with an allocation-failure hook in front of it, in the + * shape of nxt_port_test_msg_alloc_failures() (src/nxt_port_socket.c): the + * count is decremented per call, so one armed failure fires once and the + * handler then behaves exactly as it does when the process record cannot + * be allocated. The wrapper stands in front of the call rather than in + * place of the branch that follows it, so the code the test reaches stays + * the handler's own -- however this file happens to spell its response to + * a NULL process. + */ +nxt_inline nxt_process_t * +nxt_main_process_new(nxt_runtime_t *rt) +{ + if (nxt_slow_path(nxt_main_test_process_new_failure_count != 0)) { + nxt_main_test_process_new_failure_count--; + return NULL; + } + + return nxt_process_new(rt); +} + +#else +#define nxt_main_process_new(rt) nxt_process_new(rt) +#endif + const nxt_sig_event_t nxt_main_process_signals[] = { nxt_event_signal(SIGHUP, nxt_main_process_signal_handler), nxt_event_signal(SIGINT, nxt_main_process_sigterm_handler), @@ -473,28 +509,41 @@ nxt_main_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; + /* + * Every exit below the fork() is either the successful one, which lets + * the new process answer for itself, or "failed:", which answers for it. + * Nothing may leave this handler in between: START_PROCESS carries the + * stream of an RPC the router has already armed + * (nxt_router_start_app_process_handler()), and only a reply retires it. + * A silent return leaves that RPC outstanding forever -- neither + * nxt_router_app_port_ready() nor nxt_router_app_port_error() ever runs, + * so app->proto_port_requests is never cleared, every later start for + * the application parks on the prototype that is not coming, and the + * requests waiting on it are never failed. See issue #257. + */ + process = NULL; + port = rt->port_by_type[NXT_PROCESS_ROUTER]; if (nxt_slow_path(port == NULL)) { nxt_alert(task, "router port not found"); - goto close_fds; + goto failed; } if (nxt_slow_path(port->pid != nxt_recv_msg_cmsg_pid(msg))) { nxt_alert(task, "process %PI cannot start processes", nxt_recv_msg_cmsg_pid(msg)); - goto close_fds; + goto failed; } - process = nxt_process_new(rt); + process = nxt_main_process_new(rt); if (nxt_slow_path(process == NULL)) { - goto close_fds; + goto failed; } process->mem_pool = nxt_mp_create(1024, 128, 256, 32); if (process->mem_pool == NULL) { - nxt_process_use(task, process, -1); - goto close_fds; + goto failed; } process->parent_port = rt->port_by_type[NXT_PROCESS_MAIN]; @@ -622,17 +671,41 @@ nxt_main_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) failed: - nxt_process_use(task, process, -1); + if (process != NULL) { + nxt_process_use(task, process, -1); + } - port = nxt_runtime_port_find(rt, msg->port_msg.pid, + /* + * Answer on the port of the process the kernel says sent this, not the + * one the message claims to come from: msg->port_msg.pid is filled in by + * the sender (nxt_port_socket_write2()) and is not authenticated, and + * the two branches above now reach this reply before the router identity + * check has run -- or after it has failed. Keyed on the credential, the + * RPC_ERROR can only ever retire an RPC of the sender's own, so a worker + * that forges a START_PROCESS cannot use main to cancel a stream of the + * router's choosing. For a legitimate sender the two pids are equal, + * because nxt_port_socket_write2() sets port_msg.pid to its own nxt_pid; + * where SCM_CREDENTIALS is unavailable nxt_recv_msg_cmsg_pid() is + * defined as port_msg.pid, so the lookup is unchanged there. + */ + port = nxt_runtime_port_find(rt, nxt_recv_msg_cmsg_pid(msg), msg->port_msg.reply_port); if (nxt_fast_path(port != NULL)) { - nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, - -1, msg->port_msg.stream, 0, NULL); - } + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, + -1, msg->port_msg.stream, 0, NULL); -close_fds: + } else { + /* + * Nothing to answer on: the sender is gone, or never had the port it + * named. A dead sender's RPCs die with it, so this is only worth a + * diagnostic -- but a silent drop here is exactly the shape of the + * defect above, so it is not left silent. + */ + nxt_alert(task, "cannot report a failed start back: reply port %d of " + "process %PI not found", (int) msg->port_msg.reply_port, + nxt_recv_msg_cmsg_pid(msg)); + } nxt_fd_close(msg->fd[0]); msg->fd[0] = -1; @@ -642,6 +715,25 @@ nxt_main_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +#if (NXT_TESTS) + +/* + * Public wrapper that lets src/test/nxt_main_start_process_reply_test.c + * invoke the static nxt_main_start_process_handler() directly with a + * synthesised runtime and message -- used to verify that every exit above + * the fork() answers the router's START_PROCESS RPC instead of returning + * silently and stranding it (issue #257). + */ +void +nxt_main_test_run_start_process_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg) +{ + nxt_main_start_process_handler(task, msg); +} + +#endif + + static void nxt_main_process_created_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { diff --git a/src/nxt_main_process.h b/src/nxt_main_process.h index ef083d630..bb7fe68f1 100644 --- a/src/nxt_main_process.h +++ b/src/nxt_main_process.h @@ -33,5 +33,11 @@ NXT_EXPORT extern const nxt_process_init_t nxt_app_process; extern const nxt_sig_event_t nxt_main_process_signals[]; extern const nxt_sig_event_t nxt_process_signals[]; +#if (NXT_TESTS) +void nxt_main_test_process_new_failures(nxt_uint_t failures); +void nxt_main_test_run_start_process_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +#endif + #endif /* _NXT_MAIN_PROCESS_H_INCLUDED_ */ diff --git a/src/test/nxt_main_start_process_reply_test.c b/src/test/nxt_main_start_process_reply_test.c new file mode 100644 index 000000000..6432b9f21 --- /dev/null +++ b/src/test/nxt_main_start_process_reply_test.c @@ -0,0 +1,508 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for issue #257: nxt_main_start_process_handler() + * (src/nxt_main_process.c) had exits above the fork() that returned + * without answering the router. + * + * START_PROCESS carries the stream of an RPC the router has already armed + * in nxt_router_start_app_process_handler(), having incremented + * app->proto_port_requests for it. Only a reply retires that RPC: with + * none, neither nxt_router_app_port_ready() nor nxt_router_app_port_error() + * ever runs, the counter is never cleared, and every later start for the + * application parks on a prototype that is not coming. The three exits + * covered here -- no router port, a sender that is not the router, and a + * process record that cannot be allocated -- all used to "goto close_fds", + * which closes the descriptors and returns in silence. + * + * The reply is observed rather than inferred: the fixture ports carry no + * queue and leave write_ready unset, so nxt_port_socket_write() takes the + * nxt_port_msg_chk_insert() path and the message stays on port->messages + * for the test to read -- the arrangement src/test/nxt_router_start_fail_ + * test.c and src/test/nxt_router_proto_wedge_test.c already rely on. Each + * case requires exactly one message, of type RPC_ERROR and on the stream + * the incoming message named: a handler that answered twice would retire + * an RPC that is no longer the caller's. + * + * The foreign-sender case additionally pins down where the answer goes. + * The reply port is looked up by the kernel-validated sender pid, not by + * the self-declared msg->port_msg.pid, so a forged START_PROCESS can only + * ever cancel a stream of the forger's own; that case gives the two pids + * different values and requires the router's port to stay untouched. + * + * Not covered: the nxt_mp_create() failure a few lines below the process + * allocation. Arming it needs a malloc-failure hook that does not exist, + * and the sites are not independent -- both reach "failed:" through the + * same "goto", so the process-allocation case already exercises the reply + * they share. What it does not exercise is the "process != NULL" guard at + * "failed:", which only the mem_pool site can reach. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +static nxt_bool_t +nxt_main_start_process_reply_test_fd_is_open(nxt_fd_t fd) +{ + return fcntl(fd, F_GETFD) != -1; +} + + +/* + * Run the handler once and require that it answered: exactly one RPC_ERROR + * on reply_port, carrying the stream of the incoming message, and nothing + * at all on silent_port (which may be NULL when there is no second port to + * rule out). Both ports are left drained, so the next case starts from + * empty queues. + */ +static nxt_int_t +nxt_main_start_process_reply_test_case(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_port_t *reply_port, nxt_port_t *silent_port, + const char *name) +{ + nxt_fd_t fd0, fd1; + nxt_int_t ret; + nxt_uint_t n; + nxt_queue_link_t *link; + nxt_port_send_msg_t *sent; + + ret = NXT_ERROR; + sent = NULL; + n = 0; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s failed to " + "open the descriptors", name); + goto done; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_main_test_run_start_process_handler(task, msg); + + /* + * Both exits own the descriptors the message carried, and both spellings + * of the failure tail closed them -- so this is not what discriminates + * the fix, only a guard against the cleanup below closing a number that + * has already been reused. + */ + if (msg->fd[0] == -1 && !nxt_main_start_process_reply_test_fd_is_open(fd0)) + { + fd0 = -1; + } + + if (msg->fd[1] == -1 && !nxt_main_start_process_reply_test_fd_is_open(fd1)) + { + fd1 = -1; + } + + if (nxt_slow_path(fd0 != -1 || fd1 != -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s leaked a " + "descriptor", name); + goto done; + } + + for (link = nxt_queue_first(&reply_port->messages); + link != nxt_queue_tail(&reply_port->messages); + link = nxt_queue_next(link)) + { + n++; + sent = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + } + + if (nxt_slow_path(n != 1)) { + nxt_log_alert(thr->log, "main start process reply test: %s left %ui " + "replies on the reply port (expected 1) -- the " + "START_PROCESS RPC is stranded", name, n); + goto done; + } + + if (nxt_slow_path(sent->port_msg.type != _NXT_PORT_MSG_RPC_ERROR)) { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "with message type %d (expected %d, RPC_ERROR)", name, + (int) sent->port_msg.type, + (int) _NXT_PORT_MSG_RPC_ERROR); + goto done; + } + + if (nxt_slow_path(sent->port_msg.last != 1)) { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "with a non-final message, so the RPC stays armed", + name); + goto done; + } + + if (nxt_slow_path(sent->port_msg.stream != msg->port_msg.stream)) { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "on stream %uD (expected %uD)", name, + sent->port_msg.stream, msg->port_msg.stream); + goto done; + } + + if (silent_port != NULL && !nxt_queue_is_empty(&silent_port->messages)) { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "on a port of the process it was not sent by", name); + goto done; + } + + ret = NXT_OK; + +done: + + /* + * Close only what is still open: a consumed descriptor is gone already, + * and closing its number again could reach an unrelated file. + */ + if (fd0 != -1 && nxt_main_start_process_reply_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_main_start_process_reply_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + /* + * A queued message is malloc'd and takes a port reference + * (nxt_port_msg_chk_insert()). Left alone the port never reaches + * nxt_port_mp_cleanup(), so its pool leaks and the assertions there + * never run -- the same teardown src/test/nxt_router_proto_wedge_test.c + * performs, and for the same reason. + */ + nxt_port_test_run_error_handler(task, reply_port); + + if (silent_port != NULL) { + nxt_port_test_run_error_handler(task, silent_port); + } + + return ret; +} + + +/* + * Run the handler once where there is no port to answer on: the sender is a + * pid the runtime has never seen, so nxt_runtime_port_find() returns NULL + * and the handler can only log. Nothing may be queued on any port, and the + * descriptors the message carried must still be released. + */ +static nxt_int_t +nxt_main_start_process_reply_test_no_reply_case(nxt_thread_t *thr, + nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_t *port_a, + nxt_port_t *port_b, const char *name) +{ + nxt_fd_t fd0, fd1; + nxt_int_t ret; + + ret = NXT_ERROR; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s failed to " + "open the descriptors", name); + goto done; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + nxt_main_test_run_start_process_handler(task, msg); + + if (msg->fd[0] == -1 && !nxt_main_start_process_reply_test_fd_is_open(fd0)) + { + fd0 = -1; + } + + if (msg->fd[1] == -1 && !nxt_main_start_process_reply_test_fd_is_open(fd1)) + { + fd1 = -1; + } + + if (nxt_slow_path(fd0 != -1 || fd1 != -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s leaked a " + "descriptor", name); + goto done; + } + + if (nxt_slow_path(!nxt_queue_is_empty(&port_a->messages) + || (port_b != NULL + && !nxt_queue_is_empty(&port_b->messages)))) + { + nxt_log_alert(thr->log, "main start process reply test: %s answered " + "on a port, but no port belongs to the sender", name); + goto done; + } + + ret = NXT_OK; + +done: + + if (fd0 != -1 && nxt_main_start_process_reply_test_fd_is_open(fd0)) { + (void) close(fd0); + } + + if (fd1 != -1 && nxt_main_start_process_reply_test_fd_is_open(fd1)) { + (void) close(fd1); + } + + nxt_port_test_run_error_handler(task, port_a); + + if (port_b != NULL) { + nxt_port_test_run_error_handler(task, port_b); + } + + return ret; +} + +nxt_int_t +nxt_main_start_process_reply_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *router_port, *foreign_port; + nxt_runtime_t *rt, *saved_rt; + nxt_event_engine_t engine, *saved_engine; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "main start process reply test started"); + + ret = NXT_ERROR; + router_port = NULL; + foreign_port = NULL; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Only the members the reached code touches are set; the engine is what + * the queued-message teardown needs (see src/test/nxt_router_new_port_ + * test.c for the same fixture). + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* + * The router's port: both the one the handler checks the sender + * against, and -- since the router is the sender in every case but the + * forgery below -- the one the reply is written to. No queue and + * write_ready unset, so a reply is queued rather than written. + */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + router_port->pair[0] = -1; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * What nxt_runtime_port_add() does, spelled out: that helper is static + * to src/nxt_runtime.c, and it also takes a port reference the fixture + * would then have to give back. The port stays owned by this frame. + */ + rt->port_by_type[NXT_PROCESS_ROUTER] = router_port; + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, router_port) != NXT_OK)) { + goto done; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port_msg.pid = nxt_pid; + msg.port_msg.reply_port = router_port->id; + msg.port_msg.type = _NXT_PORT_MSG_START_PROCESS; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = nxt_pid; +#endif + + /* + * No router port at all. The runtime still knows the sender's port, so + * there is something to answer on -- the handler used to return without + * doing so. + */ + + rt->port_by_type[NXT_PROCESS_ROUTER] = NULL; + + msg.port_msg.stream = 0x11111111; + + ret = nxt_main_start_process_reply_test_case(thr, task, &msg, router_port, + NULL, "a missing router " + "port"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + rt->port_by_type[NXT_PROCESS_ROUTER] = router_port; + +#if (NXT_USE_CMSG_PID) + + /* + * A sender that is not the router. The message claims to come from the + * router -- port_msg.pid is filled in by the sender and is not + * authenticated -- while the credential the kernel attached names + * somebody else. The start is refused, and the refusal has to be + * reported to the process that actually sent it: answering the pid the + * message claimed would let a worker cancel a stream of the router's + * choosing. + */ + + foreign_port = nxt_port_new(task, router_port->id, nxt_pid + 1, + NXT_PROCESS_APP); + if (nxt_slow_path(foreign_port == NULL)) { + ret = NXT_ERROR; + goto done; + } + + foreign_port->pair[0] = -1; + foreign_port->pair[1] = -1; + foreign_port->socket.fd = -1; + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, foreign_port) != NXT_OK)) { + ret = NXT_ERROR; + goto done; + } + + msg.cmsg_pid = nxt_pid + 1; + msg.port_msg.stream = 0x22222222; + + ret = nxt_main_start_process_reply_test_case(thr, task, &msg, + foreign_port, router_port, + "a foreign sender"); + + msg.cmsg_pid = nxt_pid; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#endif + + /* + * The process record cannot be allocated. This is the exit furthest + * into the handler that still owes the router an answer, and the one + * that leaves "process" NULL for the guard at "failed:". + */ + + nxt_main_test_process_new_failures(1); + + msg.port_msg.stream = 0x33333333; + + ret = nxt_main_start_process_reply_test_case(thr, task, &msg, router_port, + foreign_port, + "a failed process " + "allocation"); + + nxt_main_test_process_new_failures(0); + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * Nothing to answer on. The sender is a pid the runtime knows no port + * for, so the reply-port lookup fails; the handler logs and must leave + * every queue untouched rather than answering somebody else. Both pid + * fields move together so the case holds however + * nxt_recv_msg_cmsg_pid() is defined. + */ + + msg.port_msg.pid = nxt_pid + 2; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = nxt_pid + 2; +#endif + msg.port_msg.stream = 0x44444444; + + ret = nxt_main_start_process_reply_test_no_reply_case(thr, task, &msg, + router_port, + foreign_port, + "an unknown " + "sender"); + + msg.port_msg.pid = nxt_pid; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = nxt_pid; +#endif + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "main start process reply test passed"); + + ret = NXT_OK; + +done: + + /* Whatever a failed case left armed must not follow the test out. */ + nxt_main_test_process_new_failures(0); + + if (foreign_port != NULL) { + (void) nxt_port_hash_remove(&rt->ports, foreign_port); + + nxt_port_use(task, foreign_port, -1); + } + + if (router_port != NULL) { + (void) nxt_port_hash_remove(&rt->ports, router_port); + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index f14011224..a10a8127a 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -217,6 +217,10 @@ main(int argc, char **argv) return 1; } + if (nxt_main_start_process_reply_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index ba83f3160..40878c749 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -78,6 +78,7 @@ nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); +nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); From e27b3e503e417ab84096d49ffc2c37ede96555fa Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Sat, 5 Sep 2026 15:15:15 +0100 Subject: [PATCH 080/391] http: handle file truncation in static serving My removal of the short read check in nxt_http_static_buf_completion() was a little overzealous; we still need to handle the file truncation case. Reading fewer bytes than requested can be perfectly legitimate (for example a short read from a network filesystem), so in that case we correctly carry on until we have read the expected amount of data. However, if the file has been truncated since it was stat(2)'d, then as we reach the now earlier end of file a pread(2) returns 0 while more data is still expected. With the check removed file_pos would never advance, and Unit could get stuck in a loop expecting to read the missing data. We now catch this (n == 0) and error out. Fixes: 02d1984c9 ("HTTP: Remove short read check in nxt_http_static_buf_completion()") Signed-off-by: Andrew Clayton --- src/nxt_http_static.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 7d6e9deef..d0413659e 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -1107,6 +1107,11 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) b->next = nxt_http_buf_last(r); } else { + if (nxt_slow_path(n == 0)) { + /* file truncated since it was stat(2)'d */ + nxt_http_request_error_handler(task, r, r->proto.any); + goto clean; + } fb->file_pos += n; b->next = NULL; } From 9455334742d5fd6cee3b6dd397fb592bb42f860b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 10:40:17 +0200 Subject: [PATCH 081/391] router: close the listening socket before acknowledging its removal Since the two-phase listener drain (9ae96c38, 1.36.0) a configuration that removes a listener is acknowledged as soon as accept(2) has been disarmed, so the controller can report success while the descriptor is still bound. A client that re-binds the address on seeing that success gets EADDRINUSE. That is the intended trade for a listener that has connections to drain, but it was applied to every listener, including those with nothing in flight, where there is nothing to wait for. nxt_router_listen_socket_close() (src/nxt_router.c:4116) called nxt_router_listen_socket_close_ready() before its lev->count > 1 check, so the reply was posted even when phase 2 was about to run in the same call. nxt_router_listen_socket_close_finish() (src/nxt_router.c:4212) then posted it again ahead of nxt_router_listen_socket_release(), which performs the single nxt_socket_close() of the descriptor on the last engine to reach phase 2 (src/nxt_router.c:4250-4308). Before 9ae96c38 the order in that function was release, then post; the drain moved the post ahead of the close for every listener rather than only for the ones being drained. Acknowledge in phase 1 only when the drain is real -- lev->count > 1, meaning accepted connections still hold listener references -- and let phase 2 acknowledge after the descriptor is closed otherwise. A listener with nothing in flight then keeps the pre-1.36 contract: a successful reconfiguration means the port is free. The early acknowledgement the drain feature exists for is untouched, so a listener with connections on it still does not make the control request wait for idle client timing, and test_listener_drain.py still passes. The reply is not lost or duplicated on any path. Every listener close is armed by nxt_router_listen_socket_delete(), the only writer of joint->close_job (src/nxt_router.c:4054); close() has exactly the two exits above, close_finish() has none, and the drain-completion path in nxt_router_listen_event_release() (src/nxt_router.c:4325-4345) is reachable only after a phase 1 that saw count > 1 and already posted, so its close_finish() finds joint->close_job NULL and returns early (src/nxt_router.c:4197). The joint outlives the moved call: its listener reference is dropped by nxt_router_listen_event_release() at the tail of close_finish(), and nxt_router_listen_socket_release() frees the nxt_listen_socket_t, not the socket configuration. Per engine the guarantee is that the acknowledgement follows that engine's release; the end-to-end guarantee holds because nxt_router_conf_ready() waits for every engine and the closing engine's post is ordered after its nxt_socket_close(). Verified in Docker against a --debug build with the default engine count (8 on this host) under 16x CPU load, 250 runs of test_listeners_port_release per tree, with each failure classified by whether the router's close(2) preceded the controller's reply in the debug log: master this branch acknowledged before close 10 0 other cause, correct order 4 2 harness startup error 1 1 Idle, both trees pass 100 of 100. The residual "other cause" failures have the descriptor closed and the reply posted in the correct order and occur at the same rate on master, so they are a second, pre-existing window that this change neither fixes nor introduces; they deserve their own issue. test_listeners_port_release is deliberately left as the strict guard it has always been. Co-Authored-By: Claude Opus 5 --- src/nxt_router.c | 68 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 913a137fa..c344e689c 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -4096,10 +4096,13 @@ nxt_router_worker_thread_exit(nxt_task_t *task) * * Accepting -> Draining: nxt_router_listen_socket_close() (phase 1) * disarms accept(2), marks lev->draining = 1. + * If connections are in flight it also posts the + * close_job back to the configuration thread, so + * the control request does not wait for them. * Draining -> Closed: nxt_router_listen_socket_close_finish() (phase 2) * runs once lev->count == 1 (no in-flight accepted - * connections), releases the FD and posts the - * close_job back to the configuration thread. + * connections), releases the FD and then posts the + * close_job if phase 1 did not already. * * The intermediate state lets in-flight TLS handshakes and accepted-but- * not-yet-handled connections complete cleanly instead of being RST when @@ -4152,31 +4155,36 @@ nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data) } } - /* - * Configuration is ready once accept(2) has been disarmed. The - * listener FD and old socket configuration stay alive below until - * accepted connections release their listener refs, but the control - * request must not wait for idle client timing. - */ - nxt_router_listen_socket_close_ready(joint); - /* * Phase 2 gating: wait for accepted connections to release their * refs. lev->count == 1 means only the original listener ref * remains (see nxt_listen_event() and nxt_conn_accept()). * - * Known limitation: the listening FD stays bound until the drain - * completes (nxt_router_listen_socket_release() closes it on the - * last engine's phase 2), so a configuration that re-adds the same - * address while the old listener drains fails with EADDRINUSE at - * bind time in the main process -- a visible, retryable config - * error. Closing the FD in phase 1 instead requires reworking the - * ls->count contract across engines and the nxt_process_quit() - * listen-queue walk; that belongs to the full-drain follow-up. + * When connections are still in flight the configuration is + * acknowledged here, before the FD is released: the control request + * must not wait for idle client timing. The listening FD then stays + * bound until the drain completes + * (nxt_router_listen_socket_release() closes it on the last engine's + * phase 2), so a configuration that re-adds the same address while + * the old listener drains can fail with EADDRINUSE at bind time in + * the main process -- a visible, retryable config error. Closing + * the FD in phase 1 instead requires reworking the ls->count + * contract across engines and the nxt_process_quit() listen-queue + * walk; that belongs to the full-drain follow-up. + * + * With nothing in flight there is no drain to wait for, so the reply + * is deferred to nxt_router_listen_socket_close_finish() below, + * which sends it after the descriptor is closed. That keeps the + * pre-drain contract -- a successful reconfiguration means the port + * is free -- for every listener that has no accepted connection on + * it. */ if (lev->count > 1) { nxt_debug(task, "engine %p: listen socket %d drain pending, " "in-flight: %D", engine, lev->socket.fd, lev->count - 1); + + nxt_router_listen_socket_close_ready(joint); + return; } @@ -4217,13 +4225,33 @@ nxt_router_listen_socket_close_finish(nxt_task_t *task, joint = lev->socket.data; lev->socket.data = NULL; - nxt_router_listen_socket_close_ready(joint); - /* 'task' refers to lev->task and we cannot use after nxt_free() */ task = &task->thread->engine->task; nxt_router_listen_socket_release(task, joint->socket_conf); + /* + * Acknowledge only now. nxt_router_listen_socket_release() above + * drops this engine's reference to the listen socket, and on the + * last engine to reach phase 2 it performs the single + * nxt_socket_close() of the descriptor. The configuration is not + * reported successful until every engine has acknowledged + * (nxt_router_conf_ready()), so acknowledging after the release -- + * rather than before it, as this function used to -- puts the reply + * strictly after the close for a listener with nothing in flight. + * + * A listener that did have connections to drain acknowledged early + * in phase 1 and left joint->close_job NULL, so this is the no-op it + * has always been on that path, and the still-bound descriptor + * documented there remains its known limitation. + * + * The joint outlives the call: its listener reference is dropped by + * nxt_router_listen_event_release() below, and + * nxt_router_listen_socket_release() frees the nxt_listen_socket_t, + * not the socket configuration the joint points at. + */ + nxt_router_listen_socket_close_ready(joint); + nxt_router_listen_event_release(task, lev, joint); } From 66c73dc932031528c04e32d53c7e0c4afe318843 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 10:39:13 +0200 Subject: [PATCH 082/391] test(tls): prove the session is cached before reusing it test_tls_session and test_tls_session_timeout have failed on the Alpine builders since 1.35.4 (issue #51) with "AssertionError: cache" and "AssertionError: no timeout", and a rebuild has been the only response. The cause is neither expiry nor architecture. Unit keeps no session cache of its own: nxt_ssl_session_cache() (src/nxt_openssl.c:790-802) sets the mode, size and timeout on one shared SSL_CTX. OpenSSL inserts a server session from ssl_update_cache(), which tls_finish_handshake() calls once the state machine reaches TLS_ST_OK -- after the Finished message it flushed is already on the wire. The client therefore completes its handshake fractionally before the insert. With a single router engine the event loop serialises "finish handshake, insert, accept next" and the window cannot be observed; with several, the client's immediate reconnect can be accepted by an engine that has not seen the insert yet, and a full handshake results. The tests asserted that a session is resumable in the instant it is established. That is not a property Unit offers, and it only ever passed because single-engine accept timing hid the window. So close the window on the client side instead of asserting through it: connect() now completes one HTTP request before the connection it established is reused for a resume. SSL_do_handshake() does not return on the server until ssl_update_cache() has run, and Unit reads application data only after it returns 1 (nxt_openssl_conn_handshake(), src/nxt_openssl.c:1236-1290), so a response the client has read proves the session is already in the cache. The assertions stay exactly as strict as they were: a session that genuinely fails to resume still fails the test. test_tls_session_timeout additionally stops fitting both of its halves into one window. It used timeout=4 with sleep(6), so "still cached" and "evicted" each depended on the other's margin. They are configured separately now -- a timeout no run can outlast for the first, one every run outlasts for the second -- which makes the eviction half more certain on a slow machine rather than less. The session that must expire is established after that reconfiguration, because rebuilding the TLS context flushes the cache. The comment this replaces claimed OpenSSL evaluates expiry in whole seconds. That has not been true since OpenSSL 3.2, which uses microsecond-precision OSSL_TIME, and the flake it widened the window for was this same race, mislabelled -- which is why no timeout value ever fixed it. Verified in Docker against a --debug build with the default engine count (8 on this host) under 16x CPU load: test_tls_session.py fails 5 runs in 20 on master, with the two signatures Alpine reports, and 0 runs in 20 with this change. A bounded retry loop in resume() was measured first and rejected: at cache_size=2 every retry is itself a full handshake that inserts a session and evicts the one being waited for, and it still failed 4 runs in 12 under the same load. Co-Authored-By: Claude Opus 5 --- test/test_tls_session.py | 54 +++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/test/test_tls_session.py b/test/test_tls_session.py index ea7f81bfd..b6f71b2aa 100644 --- a/test/test_tls_session.py +++ b/test/test_tls_session.py @@ -63,6 +63,28 @@ def connect(ctx=None, session=None): conn.set_session(session) conn.do_handshake() + + # Complete one request before the session is used again. OpenSSL + # inserts a server session into the shared SSL_CTX cache from + # tls_finish_handshake(), which runs after the Finished message it + # flushed has already reached the client. A handshake that has + # returned on the client therefore proves nothing about the insert: + # with more than one router engine the immediate reconnect can be + # accepted by an engine that has not seen it yet, and a full + # handshake results -- the flake tracked in issue #51. + # + # SSL_do_handshake() does not return on the server until the insert + # has happened, and Unit does not read application data before it + # returns (nxt_openssl_conn_handshake(), src/nxt_openssl.c), so a + # response the client has read proves the session is in the cache. + conn.sendall(b'GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + + response = b'' + while b'\r\n\r\n' not in response: + response += conn.recv(4096) + + assert response.startswith(b'HTTP/1.1 200'), 'request completed' + conn.shutdown() return ( @@ -119,11 +141,14 @@ def test_tls_session(): reason='session reuse is not supported', ) def test_tls_session_timeout(): - # OpenSSL evaluates session expiry in whole seconds (time_t), so a 1s - # timeout leaves barely over one second of real slack for the resume - # below and evicts the session early under CI load, making the "no - # timeout" assertion flaky. Use a comfortable window and sleep past it. - assert 'success' in add_session(cache_size=5, timeout=4) + # The two halves are configured separately rather than fitted into one + # window. "Still cached" only needs a timeout no run can outlast, and + # "evicted" only needs one every run outlasts; sharing a window made + # each assertion depend on the other's margin, and a slow machine + # broke whichever half it reached first. Configured apart, a slow + # machine makes the eviction half more certain rather than less. + + assert 'success' in add_session(cache_size=5, timeout=300) _, sess, ctx, reused = connect() assert not reused, 'new connection' @@ -131,7 +156,24 @@ def test_tls_session_timeout(): _, _, _, reused = connect(ctx, sess) assert reused, 'no timeout' - time.sleep(6) + # Three seconds, not one: the cache proof below must reconnect before + # the session expires, and a scheduling stall or an OpenSSL that counts + # this timeout in whole seconds (before 3.2) can eat a full second. + assert 'success' in add_session(cache_size=5, timeout=3) + + # The reconfiguration above rebuilds the TLS context and with it the + # session cache, so the session that has to outlive the timeout is + # established after it, not before. + _, sess, ctx, reused = connect() + assert not reused, 'new connection timeout' + + # Prove the short-timeout context cached the session at all, so the + # eviction below is attributable to the timeout and not to a session + # that was never resumable. + _, _, _, reused = connect(ctx, sess) + assert reused, 'cached before timeout' + + time.sleep(5) _, _, _, reused = connect(ctx, sess) assert not reused, 'timeout' From a3097fc7f60f427ffd5323f3fae4ce874ec5429c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 23:41:39 +0200 Subject: [PATCH 083/391] test(configuration): retry the bind in test_listeners_port_release Since the listening descriptor is closed before the removal is acknowledged, the remaining failures of this test are not an ordering defect but a property of close(2): it drops only the closing thread's reference to the open file. The router arms the same descriptor for accept(2) on every engine thread, and because those threads share one file descriptor table each such syscall takes a real reference to the socket's open file. The address leaves the kernel's bind hash at the final release of that file, which for a sibling thread is deferred to its own return to user space, so a bind(2) issued immediately after the acknowledgement can still fail with EADDRINUSE. Measured on an isolated reproducer: closing a listening descriptor while eight sibling threads call accept4(2) on it leaves the address bound for a subsequent bind(2) in 25% of 2000 rounds, while the same threads at the same syscall rate on an unrelated descriptor, and the same close with no sibling threads at all, never do (0 of 2000 each). Under Unit the observed window ranged from 65us to 95ms, with no process holding a descriptor for the socket in any of the samples. A real client re-binding a released address has to retry, so the test now retries too, for up to five seconds. A listener that is genuinely never closed still fails the test. Co-Authored-By: Claude Opus 5 --- test/test_configuration.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/test/test_configuration.py b/test/test_configuration.py index 477cbeca7..0283357a5 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -1,4 +1,5 @@ import socket +import time import pytest @@ -333,7 +334,6 @@ def test_listeners_addr_error_2(skip_alert): def test_listeners_port_release(): for _ in range(10): - fail = False with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -346,15 +346,28 @@ def test_listeners_port_release(): resp = client.conf({"listeners": {}, "applications": {}}) - try: - s.bind(('127.0.0.1', 8080)) - s.listen() - - except OSError: - fail = True - - if fail: - pytest.fail('cannot bind or listen to the address') + # The router closes the listening descriptor before it + # acknowledges the removal, but close(2) only drops the closing + # thread's reference to the open file. Every other router + # thread that had the descriptor armed for accept(2) may still + # hold a transient kernel reference to it, and the address + # leaves the bind hash only when the last one is dropped, which + # happens asynchronously. A client therefore has to retry, so + # the test retries too; a listener that is genuinely never + # closed still fails here. + deadline = time.monotonic() + 5 + + while True: + try: + s.bind(('127.0.0.1', 8080)) + s.listen() + break + + except OSError: + if time.monotonic() >= deadline: + pytest.fail('cannot bind or listen to the address') + + time.sleep(0.01) assert 'success' in resp, 'port release' From b8a7ad13308b8269014e471ef6a7b4c3e54c70ae Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 12:43:25 +0200 Subject: [PATCH 084/391] pkg(eol): sync Ubuntu 22.04/26.04 EOL dates with endoflife.date The weekly EOL check (refs #178) reports two hard errors: the matrix carried Ubuntu 22.04 as EOL 2027-04 and 26.04 as 2031-04, both earlier than the real end of standard security maintenance. Canonical's list-of-releases page (the successor of wiki.ubuntu.com/ Releases) gives Jun 2027 for 22.04 LTS and May 2031 for 26.04 LTS, and endoflife.date -- the source the validator compares against -- agrees (2027-06-01 and 2031-05-29). supported_until moves with them, keeping the documented eol + _grace_os (36 months) window. EOL.md mirrors the same two rows. Co-Authored-By: Claude Opus 5 --- EOL.md | 4 ++-- pkg/eol.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/EOL.md b/EOL.md index 997fe8a26..515f6b078 100644 --- a/EOL.md +++ b/EOL.md @@ -48,9 +48,9 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | CentOS Stream | 10 | 2030-05 | 2033-05 | 3.13 | | Amazon Linux | 2 (EOL) † | 2026-06 | 2029-06 | 3.7 ‡ | | Amazon Linux | 2023 | 2029-06 | 2032-06 | 3.11 | -| Ubuntu (LTS) | 22.04 | 2027-04 | 2030-04 | 3.10 | +| Ubuntu (LTS) | 22.04 | 2027-06 | 2030-06 | 3.10 | | Ubuntu (LTS) | 24.04 | 2029-05 | 2032-05 | 3.12 | -| Ubuntu (LTS) | 26.04 | 2031-04 | 2034-04 | 3.13 | +| Ubuntu (LTS) | 26.04 | 2031-05 | 2034-05 | 3.13 | | Debian | 11 (bullseye) (EOL) † | 2024-08 | 2027-08 | 3.9 | | Debian | 12 (bookworm) (EOL) † | 2026-07 | 2029-07 | 3.11 | | Debian | 13 (trixie) | 2028-08 | 2031-08 | 3.13 | diff --git a/pkg/eol.json b/pkg/eol.json index 7302e2ffd..a0d997548 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -22,9 +22,9 @@ {"version": "2023", "eol": "2029-06", "supported_until": "2032-06", "default_python": "3.11"} ], "ubuntu": [ - {"version": "22.04", "eol": "2027-04", "supported_until": "2030-04", "default_python": "3.10", "lts": true}, + {"version": "22.04", "eol": "2027-06", "supported_until": "2030-06", "default_python": "3.10", "lts": true}, {"version": "24.04", "eol": "2029-05", "supported_until": "2032-05", "default_python": "3.12", "lts": true}, - {"version": "26.04", "eol": "2031-04", "supported_until": "2034-04", "default_python": "3.13", "lts": true} + {"version": "26.04", "eol": "2031-05", "supported_until": "2034-05", "default_python": "3.13", "lts": true} ], "debian": [ {"version": "11", "eol": "2024-08", "supported_until": "2027-08", "default_python": "3.9", "codename": "bullseye", "note": "EOL"}, From b66ff5f34bc807cab0a826168908713538bcac2a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 14:23:17 +0200 Subject: [PATCH 085/391] pkg(eol): compare Debian against endoflife.date's support field endoflife.date redefined Debian's `eol` in September 2026: it used to carry the end of regular security support and now carries the end of LTS (`eolFrom` in the v1 API), with the old value moved to `support` (`eoasFrom`): 11 bullseye eol 2026-08-31 support 2024-08-14 12 bookworm eol 2028-06-30 support 2026-07-11 13 trixie eol 2030-06-30 support 2028-08-09 The validator read `eol` for every category, so from the 2026-09-08 run onwards it reported three Debian errors and would have had the matrix adopt the LTS dates. That contradicts the policy in EOL.md: the three-year OS extension applies to standard EOL, not to extended maintenance. Adopting them would have stretched bullseye's window from 2027-08 to 2029-08 and kept an unsupported variant in the matrix for two extra years. So the field is now chosen per category -- `support` for Debian, `eol` for everyone else -- and pkg/eol.json keeps the security-team dates it already had. A unit test pins the mapping, since the failure mode is silent: both fields parse, and the wrong one merely moves a date. EOL.md's footnote spells out the per-vendor meaning, because Ubuntu's `eol` excludes ESM while Debian's now includes LTS. Co-Authored-By: Claude Opus 5 --- EOL.md | 7 +++++++ pkg/eol/src/main.rs | 29 ++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/EOL.md b/EOL.md index 515f6b078..4dbcf1266 100644 --- a/EOL.md +++ b/EOL.md @@ -66,6 +66,13 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). † Past upstream EOL; in FreeUnit grace period. ‡ Default Python shipped by this OS is itself past upstream EOL. FreeUnit does not backport fixes to that Python version. +The "Upstream EOL" column is always the end of *standard* support, per the LTS +rule below — never an extended-maintenance date. Which endoflife.date field that +is depends on the vendor, so `pkg/eol/` picks per category: Ubuntu's `eol` is the +end of standard security maintenance (ESM excluded), while Debian's `eol` became +the end of **LTS** in September 2026, and the Debian Security Team's window — the +one this table tracks — moved to that API's `support` field. + ## Dependency Support Build-time and bundled libraries. These are **not** matrix variants — nothing in diff --git a/pkg/eol/src/main.rs b/pkg/eol/src/main.rs index d83fccc07..87bf8a4fc 100644 --- a/pkg/eol/src/main.rs +++ b/pkg/eol/src/main.rs @@ -204,6 +204,22 @@ fn fetch_api(category: &str) -> Result { // API date lookup // --------------------------------------------------------------------------- +/// endoflife.date field holding the date FreeUnit's policy is anchored to. +/// +/// EOL.md: the three-year OS extension runs from *standard* EOL, never from +/// extended maintenance. For every vendor but one that is the API's `eol`. +/// Debian is the exception: in September 2026 endoflife.date redefined Debian's +/// `eol` to the end of LTS (`eolFrom` in the v1 API) and moved the end of +/// regular security-team support — the date the policy means — to `support` +/// (`eoasFrom`). Reading `eol` for Debian would silently re-anchor the grace +/// window to LTS and keep bullseye in the matrix two years past policy. +fn api_eol_field(category: &str) -> &'static str { + match category { + "debian" => "support", + _ => "eol", + } +} + /// Result of an API EOL lookup: Ok(Some(date)), Ok(None) (version not found in API), /// or Err (network/parse failure). fn api_eol_date(category: &str, version: &str) -> Result, String> { @@ -223,7 +239,7 @@ fn api_eol_date(category: &str, version: &str) -> Result, String> None => continue, }; if cycle == version { - if let Some(eol_val) = entry.get("eol") { + if let Some(eol_val) = entry.get(api_eol_field(category)) { match eol_val { serde_json::Value::String(s) => { // Normalize to YYYY-MM (API may return YYYY-MM or YYYY-MM-DD) @@ -1149,4 +1165,15 @@ mod tests { assert_eq!(r[0].severity, Severity::Error); assert!(r[0].message.contains("unparseable")); } + + #[test] + fn debian_is_compared_against_the_support_field() { + // Debian's API `eol` became the end of LTS in September 2026; the + // policy anchors on the end of regular security support, which the API + // now calls `support`. Every other category still means `eol`. + assert_eq!(api_eol_field("debian"), "support"); + assert_eq!(api_eol_field("ubuntu"), "eol"); + assert_eq!(api_eol_field("alpine"), "eol"); + assert_eq!(api_eol_field("python"), "eol"); + } } From d1a7d33fa8645fd181b02abc3a847a5e4f1a4466 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 14:23:33 +0200 Subject: [PATCH 086/391] docs(eol): refresh the pkg/eol status snapshot for today's run The table was stamped 2026-07-13 and still listed ubuntu 22.04 at 2027-04, which this branch changes. Redone from the 2026-09-08 run: same thirteen warnings the validator prints, with the months-to-EOL columns moved on, the fedora grace windows spelled out, and a line on which API field each category is compared against. Co-Authored-By: Claude Opus 5 --- pkg/eol/README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/eol/README.md b/pkg/eol/README.md index 453a3b576..a5fda0a1b 100644 --- a/pkg/eol/README.md +++ b/pkg/eol/README.md @@ -32,27 +32,29 @@ Exit codes: `0` = clean or warnings only · `1` = errors found (drift, expiry, o Errors include the **expiry gate**: any runtime or OS entry whose `supported_until` is already in the past (EOL + grace elapsed) is a hard error and must be dropped from `pkg/eol.json`. -## Current EOL Status (2026-07-13) +## Current EOL Status (2026-09-08) All dates verified against endoflife.date API. No errors. Warnings below are proximity/past-EOL alerts only — every entry is still inside its grace window. +Dates are compared per category against the field that carries *standard* support: `support` for Debian (its `eol` became the end of LTS in September 2026), `eol` for everything else. + ### Runtimes — warnings -None. All shipped runtimes are within (or ahead of) upstream EOL; `node 20` is already flagged `(EOL)` in the matrix. +None. All shipped runtimes are within (or ahead of) upstream EOL; `go 1.25`, `node 20` and `perl 5.38` are already flagged `(EOL)` in the matrix. ### OS — warnings | Entry | Note | |-------|------| | debian 11 | Past EOL (2024-08), grace until 2027-08 | -| fedora 40, 41, 42 | Past EOL (2025-05 / 2025-12 / 2026-05), grace active | +| fedora 40, 41, 42 | Past EOL (2025-05 / 2025-12 / 2026-05), grace until 2028-05 / 2028-12 / 2029-05 | | amazonlinux 2 | Past EOL (2026-06), grace until 2029-06 | | alpine 3.20 | Past EOL (2026-04), grace until 2029-04 | -| debian 12 | EOL now (2026-07), grace until 2029-07 | -| fedora 43, 44 | EOL in ~5 / ~11 months | -| alpine 3.21, 3.22 | EOL in ~4 / ~10 months | -| centos_stream 9 | EOL in ~10 months (2027-05) | -| ubuntu 22.04 | EOL in ~9 months (2027-04) | +| debian 12 | Past EOL (2026-07), grace until 2029-07 | +| fedora 43, 44 | EOL in ~3 / ~9 months | +| alpine 3.21, 3.22 | EOL in ~2 / ~8 months | +| centos_stream 9 | EOL in ~8 months (2027-05) | +| ubuntu 22.04 | EOL in ~9 months (2027-06) | ## Architecture From aa032974f6cc6edb5a246a4eb2553a509468f1ed Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 13:07:03 +0200 Subject: [PATCH 087/391] build: probe memfd_create() without the kernel UAPI header auto/shmem:110-130 included in the memfd_create() probe, which tests for the distribution's kernel headers package rather than for the syscall. Alpine ships that header in "linux-headers", not in musl-dev, so an ordinary "apk add build-base" build reported checking for memfd_create() ... not found and libunit fell back to shm_open() (src/nxt_unit.c:4098-4109). shm_open() resolves through /dev/shm, which does not exist inside an isolation.rootfs and which Unit never automounts, so every rootfs application died in nxt_unit_init() (src/nxt_unit.c:605-608) with [unit] shm_open(/unit..) failed: No such file or directory (2) MFD_CLOEXEC and the memfd_create() declaration are in under _GNU_SOURCE on glibc >= 2.27 and musl >= 1.1.20, so probe that first and keep the variant only as a fallback for older libc, recording it as NXT_HAVE_LINUX_MEMFD_H. The consumers pick the same header the probe validated: src/nxt_port_memory.c:9-17 (which gets _GNU_SOURCE via nxt_main.h -> src/nxt_unix.h:26-27), src/nxt_unit.c:18-24, and the three C tests that call SYS_memfd_create directly. .github/workflows/build-distros.yml installs linux-headers on the alpine-edge leg (:161) and so could never see this; add a minimal Alpine leg that builds without the package and asserts the probe line, since a build with the shm_open() fallback is still green. Fixes #273. Co-Authored-By: Claude Opus 5 --- .github/workflows/build-distros.yml | 46 ++++++++++++++++++++++++++- auto/shmem | 37 ++++++++++++++++++++- src/nxt_port_memory.c | 4 +++ src/nxt_unit.c | 4 +++ src/test/nxt_port_ready_test.c | 4 ++- src/test/nxt_router_new_port_test.c | 4 ++- src/test/nxt_unit_port_recv_test.c | 2 +- test/syscalls/README.md | 13 +++++--- test/syscalls/baseline-linux-musl.txt | 1 + 9 files changed, 105 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 214693519..e75ce5794 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -221,13 +221,57 @@ jobs: run: make wasm-wasi-component if: matrix.compiler == 'gcc' + # Alpine without "linux-headers". The full alpine-edge leg installs it (the + # Rust/bindgen and JDK steps are happier with the kernel UAPI headers around), + # and that is exactly what masked #273 for as long as it existed: the + # memfd_create() probe used to include , which Alpine ships in + # linux-headers rather than in musl-dev, so an ordinary "apk add build-base" + # build silently fell back to shm_open() and every isolation.rootfs app died. + # This leg is deliberately minimal -- core only, no modules -- so that it + # tests the probe and nothing else, and it asserts the probe result instead of + # trusting a green build, which would stay green with the fallback in place. + alpine-no-linux-headers: + runs-on: ubuntu-latest + timeout-minutes: 20 + + container: + image: alpine:edge + + steps: + - name: Install tools/deps + run: | + apk update && apk upgrade + apk add gcc make musl-dev openssl-dev pcre2-dev + if apk info -e linux-headers; then + echo "linux-headers must not be installed on this leg" >&2 + exit 1 + fi + + - uses: actions/checkout@v5 + + - name: configure unit + run: | + set -o pipefail + ./configure --openssl --tests | tee configure.log + + - name: assert memfd_create() was detected + run: | + grep -q 'checking for memfd_create() \.\.\. found' configure.log || { + echo 'memfd_create() not detected without linux-headers (#273)' >&2 + grep -n 'memfd' configure.log build/autoconf.err >&2 || true + exit 1 + } + + - name: make unit + run: make -j 4 + # A weekly red check that nobody caused is a check nobody reads, so a # scheduled failure is reported as a tracking issue instead, the same way # eol-check.yml handles its weekly run. report: runs-on: ubuntu-latest timeout-minutes: 10 - needs: [ fedora-rawhide, alpine-edge ] + needs: [ fedora-rawhide, alpine-edge, alpine-no-linux-headers ] if: failure() && github.event_name == 'schedule' steps: diff --git a/auto/shmem b/auto/shmem index cc85d1408..421943be3 100644 --- a/auto/shmem +++ b/auto/shmem @@ -109,12 +109,20 @@ fi # Linux +# MFD_CLOEXEC and the memfd_create() declaration live in under +# _GNU_SOURCE on glibc >= 2.27 and on musl >= 1.1.20. Probing +# instead tests for the kernel UAPI headers package rather than for the +# syscall: Alpine ships that header in "linux-headers", not in musl-dev, so a +# plain "apk add build-base" build used to fall back to shm_open(), which then +# fails inside an isolation.rootfs because /dev/shm does not exist there. + nxt_feature="memfd_create()" nxt_feature_name=NXT_HAVE_MEMFD_CREATE nxt_feature_run=yes nxt_feature_incs= nxt_feature_libs= -nxt_feature_test="#include +nxt_feature_test="#define _GNU_SOURCE + #include #include #include @@ -130,6 +138,33 @@ nxt_feature_test="#include . auto/feature +if [ $nxt_found = no ]; then + + # Older libc: MFD_CLOEXEC is only in the kernel UAPI header. + + nxt_feature="memfd_create() in " + nxt_feature_test="#include + #include + #include + + int main(void) { + static char name[] = \"/unit.configure\"; + + int fd = syscall(SYS_memfd_create, name, + MFD_CLOEXEC); + if (fd == -1) + return 1; + + return 0; + }" + . auto/feature + + if [ $nxt_found = yes ]; then + nxt_have=NXT_HAVE_LINUX_MEMFD_H . auto/have + fi +fi + + if [ "$nxt_shm_open_found$nxt_found" = nono ]; then $echo $echo $0: error: no shared memory implementation found. diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 1808bddb7..02dcb480f 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -8,7 +8,11 @@ #if (NXT_HAVE_MEMFD_CREATE) +#if (NXT_HAVE_LINUX_MEMFD_H) #include +#else +#include +#endif #include #include diff --git a/src/nxt_unit.c b/src/nxt_unit.c index cae7ecd09..296b10bb3 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -17,7 +17,11 @@ #include "nxt_websocket.h" #if (NXT_HAVE_MEMFD_CREATE) +#if (NXT_HAVE_LINUX_MEMFD_H) #include +#else +#include +#endif #endif #define NXT_UNIT_MAX_PLAIN_SIZE 1024 diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index 327041e77..f30801d58 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -38,8 +38,10 @@ #include #include -#if (NXT_HAVE_MEMFD_CREATE) +#if (NXT_HAVE_MEMFD_CREATE && NXT_HAVE_LINUX_MEMFD_H) #include +#endif +#if (NXT_HAVE_MEMFD_CREATE) #include #endif diff --git a/src/test/nxt_router_new_port_test.c b/src/test/nxt_router_new_port_test.c index 7af5644bf..f9e13f59d 100644 --- a/src/test/nxt_router_new_port_test.c +++ b/src/test/nxt_router_new_port_test.c @@ -40,8 +40,10 @@ #include #include -#if (NXT_HAVE_MEMFD_CREATE) +#if (NXT_HAVE_MEMFD_CREATE && NXT_HAVE_LINUX_MEMFD_H) #include +#endif +#if (NXT_HAVE_MEMFD_CREATE) #include #endif diff --git a/src/test/nxt_unit_port_recv_test.c b/src/test/nxt_unit_port_recv_test.c index c798177d1..cf05111f4 100644 --- a/src/test/nxt_unit_port_recv_test.c +++ b/src/test/nxt_unit_port_recv_test.c @@ -44,7 +44,7 @@ #include #include -#if (NXT_HAVE_MEMFD_CREATE) +#if (NXT_HAVE_MEMFD_CREATE && NXT_HAVE_LINUX_MEMFD_H) #include #endif diff --git a/test/syscalls/README.md b/test/syscalls/README.md index 800d51c2a..6b8be43e3 100644 --- a/test/syscalls/README.md +++ b/test/syscalls/README.md @@ -90,11 +90,14 @@ would then fail on a language patch release, which is noise, not drift. Covering the module path is a separate job with a separate baseline; see the follow-ups in the pull request that added this. -The two lists are not directly comparable with each other: 11 syscalls appear -only in the glibc baseline and 6 only in the musl one, because the libcs pick -different syscalls for the same operation (`epoll_create` vs `epoll_create1`, -`clone3` vs `fork`, `openat` vs `open`, and `memfd_create` only on glibc). -That is the reason both are captured. +The two lists are not directly comparable with each other: 10 syscalls appear +only in the glibc baseline (`access`, `clone3`, `epoll_create`, `epoll_wait`, +`lseek`, `newfstatat`, `openat`, `prlimit64`, `rseq`, `set_robust_list`) and 6 +only in the musl one (`epoll_create1`, `epoll_pwait`, `fork`, `futex`, +`membarrier`, `open`), because the libcs pick different syscalls for the same +operation (`epoll_create` vs `epoll_create1`, `clone3` vs `fork`, `openat` vs +`open`). `memfd_create` appears on both now that the probe finds it on musl +without the kernel UAPI header. That is the reason both are captured. ## Guards diff --git a/test/syscalls/baseline-linux-musl.txt b/test/syscalls/baseline-linux-musl.txt index 6caed05d6..680d3daec 100644 --- a/test/syscalls/baseline-linux-musl.txt +++ b/test/syscalls/baseline-linux-musl.txt @@ -34,6 +34,7 @@ gettid ioctl listen membarrier +memfd_create mkdir mmap mprotect From beb54cbd43f89eb6480cfe783dc84bfcdfb3e70a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 6 Sep 2026 16:40:11 +0200 Subject: [PATCH 088/391] router: hold a temp conf reference for the engine quit job nxt_router_engine_quit() (src/nxt_router.c:3657) allocates its nxt_joint_job_t -- which embeds both the nxt_work_t and the nxt_task_t that work item points at -- from tmcf->mem_pool, but set job->tmcf to NULL and took no tmcf->count reference, unlike every other joint job (nxt_router_engine_joints_create() and _joints_delete() both do tmcf->count++). nxt_router_engines_post() hands that work item to nxt_event_engine_post(), i.e. onto the dying engine's locked post queue, where it sits until that engine's thread drains it. With nothing holding the temporary configuration, nxt_router_conf_ready() (:1381) can then reach nxt_mp_release(tmcf->mem_pool) first, and the worker dereferences freed memory two ways: work->task->thread = thr in nxt_locked_work_queue_move() (src/nxt_work_queue.c:304), and task->log in the nxt_debug() at the head of nxt_router_worker_thread_quit() -- a router SIGSEGV, or an ASan heap-use-after-free whose allocation frame is nxt_router_engine_quit(). Take the reference at allocation and give the handler the job back (job->work.obj = job), so it can post an nxt_router_conf_wait() to the configuration thread and hand the reference back the way nxt_router_listen_socket_create()/_update() already do. The post is the last use of the job: the configuration thread may release the pool immediately afterwards, so the possible thread exit that follows continues on the engine's own task rather than on the pool-allocated one (nxt_router_thread_start() sets engine->task.thread). That re-post idiom now had four copies, and the work.next = NULL in it is load-bearing -- the job is still chained on recf->jobs, so posting without the reset splices its old neighbour into the target engine's locked queue, silently. Factor it into nxt_router_conf_wait_post() (:1390) and use it at all four sites; no behaviour change. The window is widest with no listeners configured: engine->joints is then empty, so nxt_router_worker_thread_quit() goes straight to nxt_router_worker_thread_exit() with no listen-socket delete job on the same queue to serialise behind. The regression test drives that shape -- listen_threads 8 -> 4 -> 8 with "listeners": {}, 50 cycles -- and fails 8 of 10 runs on master, 10 of 10 against an ASan build, and 0 of 10 here (0 of 10 under ASan too). Fixes #186. Co-Authored-By: Claude Opus 5 --- src/nxt_router.c | 58 +++++++++++++++++++++++++++---------- test/test_listen_threads.py | 41 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index c344e689c..1728011a8 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1377,6 +1377,26 @@ nxt_router_conf_wait(nxt_task_t *task, void *obj, void *data) } +/* + * Hand a joint job's temporary configuration reference back to the + * configuration thread. Resetting work.next is not optional: the job was + * chained on recf->jobs when it was built, and posting a work item that still + * points at its old neighbour silently splices that neighbour into the target + * engine's locked queue. The job must not be touched after the post -- the + * configuration thread may run nxt_router_conf_wait() and release the pool the + * job lives in at any point from here on. + */ + +nxt_inline void +nxt_router_conf_wait_post(nxt_joint_job_t *job) +{ + job->work.next = NULL; + job->work.handler = nxt_router_conf_wait; + + nxt_event_engine_post(job->tmcf->engine, &job->work); +} + + static void nxt_router_conf_ready(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) { @@ -3670,9 +3690,15 @@ nxt_router_engine_quit(nxt_router_temp_conf_t *tmcf, job->task = tmcf->engine->task; job->work.handler = nxt_router_worker_thread_quit; job->work.task = &job->task; - job->work.obj = NULL; + job->work.obj = job; job->work.data = NULL; - job->tmcf = NULL; + job->tmcf = tmcf; + + /* + * The job outlives this call on the target engine's locked queue, so hold + * the pool it lives in, as the joints create/delete jobs do. + */ + tmcf->count++; return NXT_OK; } @@ -3963,10 +3989,7 @@ nxt_router_listen_socket_create(nxt_task_t *task, void *obj, void *data) ls->count++; nxt_thread_spin_unlock(lock); - job->work.next = NULL; - job->work.handler = nxt_router_conf_wait; - - nxt_event_engine_post(job->tmcf->engine, &job->work); + nxt_router_conf_wait_post(job); } @@ -4017,10 +4040,7 @@ nxt_router_listen_socket_update(nxt_task_t *task, void *obj, void *data) lev->socket.data = joint; lev->listen = joint->socket_conf->listen; - job->work.next = NULL; - job->work.handler = nxt_router_conf_wait; - - nxt_event_engine_post(job->tmcf->engine, &job->work); + nxt_router_conf_wait_post(job); /* * The task is allocated from configuration temporary @@ -4063,16 +4083,27 @@ nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data) static void nxt_router_worker_thread_quit(nxt_task_t *task, void *obj, void *data) { + nxt_joint_job_t *job; nxt_event_engine_t *engine; nxt_debug(task, "router worker thread quit"); + job = obj; + engine = task->thread->engine; engine->shutdown = 1; + /* + * Give the reference nxt_router_engine_quit() took back. The task this + * handler was called with is allocated from the same pool as the job, so + * the exit below has to continue on the engine's own task. + */ + + nxt_router_conf_wait_post(job); + if (nxt_queue_is_empty(&engine->joints)) { - nxt_router_worker_thread_exit(task); + nxt_router_worker_thread_exit(&engine->task); } } @@ -4204,10 +4235,7 @@ nxt_router_listen_socket_close_ready(nxt_socket_conf_joint_t *joint) joint->close_job = NULL; - job->work.next = NULL; - job->work.handler = nxt_router_conf_wait; - - nxt_event_engine_post(job->tmcf->engine, &job->work); + nxt_router_conf_wait_post(job); } diff --git a/test/test_listen_threads.py b/test/test_listen_threads.py index 6c258524f..63730c3b0 100644 --- a/test/test_listen_threads.py +++ b/test/test_listen_threads.py @@ -103,3 +103,44 @@ def test_listen_threads_engine_fds(): f'router leaked {after - before} descriptors over a listen_threads' f' {THREADS} -> 1 -> {THREADS} cycle ({before} before, {after} after)' ) + + +def test_listen_threads_shrink_grow_no_listeners(): + # nxt_router_engine_quit() allocates its nxt_joint_job_t (the nxt_work_t + # *and* the nxt_task_t the work item points at) from tmcf->mem_pool but + # used to take no tmcf->count reference, so nxt_router_conf_ready() could + # release the pool while the job was still sitting on the dying engine's + # locked work queue. The window is widest with no listeners configured: + # engine->joints is then empty, so nxt_router_worker_thread_quit() is the + # only thing standing between the post and nxt_thread_exit(), and there is + # no listen-socket delete job on the same queue to serialise behind. + # + # Each PUT is asserted, so a "Value doesn't exist." reply is never counted + # as a clean cycle, and the router pid is compared every cycle because the + # control API only reports the failed apply of the crashed generation, not + # the crash itself. + # + # 50 cycles, not fewer: the per-cycle crash probability is only ~3%, so the + # count is what buys the detection rate. Measured on master, 10 runs each: + # 50 cycles caught it 8 times in 1.2 s per run, 20 cycles 4 times in 0.8 s. + # Halving the detection to save 0.4 s is not a trade worth making; the run + # time here is dominated by pytest and unitd start-up either way. + assert 'success' in client.conf( + { + "listeners": {}, + "routes": [{"action": {"return": 200}}], + } + ) + + pid = pid_by_name('unit: router') + + for i in range(50): + for threads in (8, 4, 8): + assert 'success' in client.conf( + {"listen_threads": threads}, 'settings' + ), f'cycle {i}: listen_threads {threads}' + + assert pid_by_name('unit: router') == pid, ( + f'router restarted during cycle {i}: the engine-quit job was' + ' freed with tmcf->mem_pool before the dying engine drained it' + ) From 5ff0492a525664615b92ce2cda73cf1161770c3b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 15:03:14 +0200 Subject: [PATCH 089/391] http: release the application's buffers after compressing a response nxt_http_comp_compress_app_response() (src/nxt_http_compression.c:170) released the buffer it had just compressed with nxt_buf_free(), which is "#define nxt_buf_free(mp, b) nxt_mp_free((mp), (b))" (src/nxt_buf.h:239): a plain pool free that never runs b->completion_handler. For a buffer that came from the application through shared memory -- the only kind this function touches -- that handler is nxt_port_mmap_buf_completion() (src/nxt_port_memory.c:159), and it is the only thing that marks the buffer's chunks free in hdr->free_map, drops the nxt_port_mmap_handler_t reference and releases the nxt_mp_retain() on port->mem_pool. So every compressed response leaked the shared memory it was built from. That is fatal rather than merely wasteful: an application worker may hold only shm_limit bytes of shared memory (100 MB by default), so after enough compressed bytes nxt_unit_get_outgoing_buf() can no longer allocate and the worker stops answering. Measured with a PHP application returning a 9 MB incompressible body over gzip: the eleventh request is the last one served, and the twelfth never gets a response. Under ASan the router additionally leaks the nxt_port_mmap_handler_t allocations from nxt_port_incoming_port_mmap() (src/nxt_port_memory.c:344), which the fixed build does not. Release each consumed buffer through its own completion handler instead, posted to the engine's fast work queue. in->next is cleared first, because nxt_port_mmap_buf_completion() walks the chain it is handed. While here, walk the whole chain rather than the head alone. nxt_port_mmap_read() (src/nxt_port_memory.c:911) makes one nxt_buf_t per nxt_port_mmap_msg_t and the call site links the result into r->out with nxt_buf_chain_add() (src/nxt_router.c:4531), yet the function compressed the head and then replaced the entire chain with that single output buffer, so any buffer behind the head was dropped from the response and ctx->clen_sent -- which decides whether the compressor is told it is finishing the stream -- counted only the head. No reproducer for that: libunit sends one nxt_port_mmap_msg_t per port message (nxt_unit_mmap_buf_send(), src/nxt_unit.c:2854), and instrumenting the function showed a chain length of exactly 1 on every call, including under a throttled client, so the drop is not reachable from an application response today. It is still what the code says, and a chain from any other producer would hit it, so this is fixed rather than relied upon. Non-port-mmap buffers -- the trailing sync buffer, and a plain-mode response -- are passed through untouched, which is the same condition the old code tested on the head. The output buffer gets a small fixed margin on top of bound(), because the flush marker each compressor emits per call is not part of what bound() promises for the input alone. Tests: test_php_compression_shm_not_leaked fails on master (the request that follows shm exhaustion times out) and passes here; test_php_compression_large_random_body covers the chain walk for gzip, br and zstd. Fixes: #177 Co-Authored-By: Claude Opus 5 --- src/nxt_http_compression.c | 121 ++++++++++++++------ test/php/comp_random_body/index.php | 14 +++ test/test_php_compression.py | 164 ++++++++++++++++++++++++++-- 3 files changed, 256 insertions(+), 43 deletions(-) create mode 100644 test/php/comp_random_body/index.php diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 646cdfb95..08460e6fe 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -18,6 +18,9 @@ #define NXT_COMP_LEVEL_UNSET INT8_MIN +/* Headroom for the flush marker a compressor emits per call. */ +#define NXT_HTTP_COMP_FLUSH_SLACK 64 + typedef enum nxt_http_comp_scheme_e nxt_http_comp_scheme_t; typedef struct nxt_http_comp_type_s nxt_http_comp_type_t; @@ -174,7 +177,7 @@ nxt_http_comp_compress_app_response(nxt_task_t *task, nxt_http_request_t *r, bool last; size_t buf_len; ssize_t cbytes; - nxt_buf_t *buf; + nxt_buf_t *in, *next, *buf, *out, **tail; nxt_off_t in_len; nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); @@ -182,51 +185,103 @@ nxt_http_comp_compress_app_response(nxt_task_t *task, nxt_http_request_t *r, return NXT_OK; } - if (!nxt_buf_is_port_mmap(*b)) { - return NXT_OK; - } + /* + * What arrives here is a chain, not a single buffer: + * nxt_port_mmap_read() (src/nxt_port_memory.c) makes one nxt_buf_t per + * nxt_port_mmap_msg_t, the call site links it into r->out with + * nxt_buf_chain_add(), and a sync buffer may sit at the tail. Walk it. + * + * The previous version compressed the head and then replaced the whole + * chain with that one output buffer, so anything behind the head was + * dropped from the response and never released, and ctx->clen_sent + * counted only the head -- which is what decides whether the compressor + * is told it is finishing the stream. + */ - in_len = (*b)->mem.free - (*b)->mem.pos; - buf_len = nxt_http_comp_bound(in_len); + out = NULL; + tail = &out; - buf = nxt_buf_mem_ts_alloc(task, (*b)->data, buf_len); - if (nxt_slow_path(buf == NULL)) { - return NXT_ERROR; - } + for (in = *b; in != NULL; in = next) { + next = in->next; + in->next = NULL; - buf->data = (*b)->data; + /* + * Buffers that did not come through shared memory are passed + * through untouched: the trailing sync/last buffer carries no data, + * and a plain-mode response is not compressed at all (the same + * condition the single-buffer version tested on the head). + */ + if (!nxt_buf_is_port_mmap(in)) { + *tail = in; + tail = &in->next; + continue; + } - last = ctx->clen_sent + in_len == ctx->resp_clen; + in_len = in->mem.free - in->mem.pos; - cbytes = nxt_http_comp_compress(buf->mem.start, buf_len, - (*b)->mem.pos, in_len, last); - if (cbytes == -1) { - nxt_buf_free(buf->data, buf); - return NXT_ERROR; - } + last = ctx->clen_sent + in_len == ctx->resp_clen; + + if (in_len == 0 && !last) { + goto release; + } - buf->mem.free += cbytes; + /* + * The per-call flush marker each compressor emits after the input + * is not part of what bound() promises for the input alone, so the + * output buffer gets a small fixed margin on top. + */ + buf_len = nxt_http_comp_bound(in_len) + NXT_HTTP_COMP_FLUSH_SLACK; + + buf = nxt_buf_mem_ts_alloc(task, in->data, buf_len); + if (nxt_slow_path(buf == NULL)) { + goto fail; + } - ctx->clen_sent += in_len; + cbytes = nxt_http_comp_compress(buf->mem.start, buf_len, + in->mem.pos, in_len, last); + if (nxt_slow_path(cbytes == -1)) { + nxt_buf_free(buf->data, buf); + goto fail; + } -#define nxt_swap_buf(db, sb) \ - do { \ - nxt_buf_t **db_ = (db); \ - nxt_buf_t **sb_ = (sb); \ - nxt_buf_t *tmp_; \ - \ - tmp_ = *db_; \ - *db_ = *sb_; \ - *sb_ = tmp_; \ - } while (0) + buf->mem.free += cbytes; - nxt_swap_buf(b, &buf); + ctx->clen_sent += in_len; -#undef nxt_swap_buf + *tail = buf; + tail = &buf->next; - nxt_buf_free(buf->data, buf); + release: + + /* + * The compressed bytes have been copied out, so the shared memory + * chunk can go back to the application. This has to run the + * buffer's own completion handler -- nxt_buf_free() is a plain pool + * free, and using it here (as the single-buffer version did) leaked + * the chunk, the mmap_handler reference and the port mem_pool + * reference on every compressed response. in->next was cleared + * above because nxt_port_mmap_buf_completion() walks the chain. + */ + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + in->completion_handler, task, in, in->parent); + } + + *b = out; return NXT_OK; + +fail: + + /* + * Keep whatever has been produced so far, and re-attach the input that + * has not been consumed, so that the caller's error path owns the whole + * chain and releases it. + */ + in->next = next; + *tail = in; + *b = out; + + return NXT_ERROR; } diff --git a/test/php/comp_random_body/index.php b/test/php/comp_random_body/index.php new file mode 100644 index 000000000..faa871729 --- /dev/null +++ b/test/php/comp_random_body/index.php @@ -0,0 +1,14 @@ + 100 MB in total: more shared memory than an app may hold + + +def raw_get(path, encoding, timeout=30): + """ + A plain socket request. The shared client cannot express "fail rather + than block forever", which is exactly the failure these tests look for. + """ + sock = socket.create_connection(('127.0.0.1', 8080), timeout) + sock.settimeout(timeout) + + try: + sock.sendall( + f'GET {path} HTTP/1.1\r\n' + f'Host: localhost\r\n' + f'Accept-Encoding: {encoding}\r\n' + f'Connection: close\r\n\r\n'.encode() + ) + + chunks = [] + while True: + data = sock.recv(256 * 1024) + if not data: + break + chunks.append(data) + + finally: + sock.close() + + resp = b''.join(chunks) + head, _, body = resp.partition(b'\r\n\r\n') + lines = head.split(b'\r\n') + + status = int(lines[0].split()[1]) + + headers = {} + for line in lines[1:]: + name, _, value = line.partition(b':') + headers[name.strip().decode()] = value.strip().decode() + + if headers.get('Transfer-Encoding') == 'chunked': + body = dechunk(body) + + return status, headers, body + + +def decompress(encoding, data): + if encoding == 'gzip': + return gzip.decompress(data) + + if encoding == 'br': + brotli = pytest.importorskip( + 'brotli', reason='python brotli bindings not installed' + ) + return brotli.decompress(data) + + if encoding == 'zstd': + try: + from compression import zstd # Python >= 3.14 + + return zstd.decompress(data) + except ImportError: + zstandard = pytest.importorskip( + 'zstandard', reason='python zstd bindings not installed' + ) + return zstandard.ZstdDecompressor().decompressobj().decompress(data) + + pytest.fail(f'no decompressor for {encoding}') + + +@pytest.mark.parametrize('encoding', ['gzip', 'br', 'zstd']) +def test_php_compression_large_random_body(encoding): + """ + Coverage for the chain walk in nxt_http_comp_compress_app_response(): + a multi-megabyte incompressible body arrives as a long sequence of + port-mmap buffers, so every compressor is driven across many calls with + the finishing one arriving last. Check the body still round-trips. + """ + client.load('comp_random_body') + configure_compression(encoding) + + status, headers, body = raw_get( + f'/?n={RANDOM_BODY_SIZE}', encoding + ) + + assert status == 200, 'status' + assert headers['Content-Encoding'] == encoding, 'encoding advertised' + + # The compressed body is framed either by Content-Length or by chunked + # transfer encoding -- never by both, and never by neither, or the client + # cannot tell where it ends. + has_clen = 'Content-Length' in headers + is_chunked = headers.get('Transfer-Encoding') == 'chunked' + + assert has_clen != is_chunked, f'exactly one framing: {headers}' + + if has_clen: + assert len(body) == int(headers['Content-Length']), 'framing consistent' + + plain = decompress(encoding, body) + + assert len(plain) == RANDOM_BODY_SIZE, 'full body length' + assert ( + hashlib.sha256(plain).hexdigest() == headers['X-Body-Sha256'] + ), 'round-trips byte for byte' + + +def test_php_compression_shm_not_leaked(): + """ + Regression test for #177. + + nxt_http_comp_compress_app_response() released the application's buffer + with nxt_buf_free(), a plain pool free that never runs the buffer's + completion handler. For a port-mmap buffer that handler, + nxt_port_mmap_buf_completion(), is what marks the shared memory chunks + free again, so every compressed response leaked its chunks. + + An application may hold only so much shared memory (shm_limit, 100 MB by + default), so the leak is fatal rather than merely wasteful: after enough + compressed bytes the worker can no longer allocate a buffer and the next + request never gets a response. Before the fix this wedges partway + through the loop and the request below times out. + """ + client.load('comp_random_body') + configure_compression() + + for i in range(LEAK_REQUESTS): + status, headers, body = raw_get( + f'/?n={RANDOM_BODY_SIZE}', 'gzip' + ) + + assert status == 200, f'status of request {i}' + assert headers['Content-Encoding'] == 'gzip', f'encoding, request {i}' + assert ( + hashlib.sha256(gzip.decompress(body)).hexdigest() + == headers['X-Body-Sha256'] + ), f'body of request {i}' From 4aaede2781f367863bea4eee8d346247be3f06c6 Mon Sep 17 00:00:00 2001 From: x Date: Sat, 5 Sep 2026 15:03:50 +0200 Subject: [PATCH 090/391] http: drop the response body on a status that defines none An application that returned 204 or 304 and also wrote a body had those bytes forwarded verbatim, with no Content-Length and no chunked framing, so a downstream parser on a keep-alive connection read them as the start of the next response -- a response-smuggling primitive an application could shape deliberately. src/nxt_h1proto.c:1459 declined chunked framing for NXT_HTTP_NOT_MODIFIED and NXT_HTTP_NO_CONTENT, and those two lines were the only places in src/*.c that named either status. Nothing then dropped the body, so nxt_http_request_send() handed the application's buffers straight to the connection. The same held for every response to HEAD, which had no special handling outside src/nxt_http_static.c:313. Decide it once, in the layer every response source -- application, proxy, static, "return" -- passes through: - nxt_http_request_header_send() (src/nxt_http_request.c:686) now sets r->no_body via nxt_http_request_is_bodyless(): 1xx, 204, 304, or any response to HEAD, excluding a 101 WebSocket upgrade, whose following bytes are the upgraded protocol rather than a message body. The status is tested alongside websocket_handshake, not the flag alone: that flag is set while the request headers are parsed, long before the application picks a status, and the h1 sender only treats the response as an upgrade when it is also 101. Testing the flag by itself would leave an application answering a WebSocket-upgrade request with 204 plus a body on the unframed path. - For 1xx and 204 it also drops the framing headers, per RFC 9110 Sect. 8.6 and RFC 9112 Sect. 6.1: content_length_n is reset, and nxt_http_request_drop_framing_fields() marks every Content-Length and Transfer-Encoding response field skipped, matching names case-insensitively. r->resp.content_length alone is not enough -- "response_headers" adds fields through nxt_http_resp_field_zero_add() without setting that pointer, and an application-supplied Transfer-Encoding is a generic field too. Both would otherwise be serialized verbatim and desync the next response. A 304 keeps Content-Length -- there it describes the body the client already has -- and so does a HEAD response. - nxt_http_request_send() (src/nxt_http_request.c) routes a bodyless response through nxt_http_request_body_drop(), which unlinks the data-carrying buffers and nxt_sendbuf_drain()s them so their completion handlers run and the shared memory they hold is released. Sync buffers are left untouched -- one may be allocated with NXT_BUF_SYNC_SIZE, so its mem.free must not even be read -- and a buffer that also carries the "last" marker stays in the chain but is emptied in place, so the request still completes normally. - src/nxt_h1proto.c:1459 additionally tests r->no_body, so HEAD and 1xx stop getting chunked framing too. Without that a discarded HEAD body would still have emitted a terminating 0\r\n\r\n chunk. The pre-HTTP/1.1 branch there no longer disables keep-alive for a bodyless response: a close is only needed to delimit a body, and there is none. Keep-alive is preserved: the header is framed correctly and nothing follows it, so the next request on the connection parses cleanly. An application doing this is common and harmless once dropped, so it is logged at debug level only. Fixes #164 Co-Authored-By: Claude Opus 5 --- src/nxt_h1proto.c | 15 +- src/nxt_http.h | 1 + src/nxt_http_request.c | 178 ++++++++++++++++++ test/php/no_body_status/index.php | 15 ++ test/test_http_no_body_status.py | 289 ++++++++++++++++++++++++++++++ 5 files changed, 496 insertions(+), 2 deletions(-) create mode 100644 test/php/no_body_status/index.php create mode 100644 test/test_http_no_body_status.py diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 57d8cb9b2..1900b4df8 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -1456,7 +1456,13 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, if (r->resp.content_length == NULL || r->resp.content_length->skip) { if (http11) { - if (n != NXT_HTTP_NOT_MODIFIED + /* + * r->no_body already covers 204 and 304, plus 1xx and every + * response to HEAD; the two status tests are kept so the + * framing rule stays readable at the point it is applied. + */ + if (!r->no_body + && n != NXT_HTTP_NOT_MODIFIED && n != NXT_HTTP_NO_CONTENT && body_handler != NULL && !h1p->websocket) @@ -1467,7 +1473,12 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, size -= nxt_length("\r\n"); } - } else { + } else if (!r->no_body) { + /* + * A pre-HTTP/1.1 client needs the close to delimit a body; + * a response that has no body needs no such delimiter, so + * keep-alive stays as negotiated. + */ h1p->keepalive = 0; } } diff --git a/src/nxt_http.h b/src/nxt_http.h index df24e7ebc..b3974d8a9 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -267,6 +267,7 @@ struct nxt_http_request_s { uint8_t error; /* 1 bit */ uint8_t websocket_handshake; /* 1 bit */ uint8_t chunked; /* 1 bit */ + uint8_t no_body; /* 1 bit */ }; diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index c15abd99d..91287ea73 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -22,6 +22,10 @@ static void nxt_http_request_forward_protocol(nxt_http_request_t *r, static void nxt_http_request_ready(nxt_task_t *task, void *obj, void *data); static void nxt_http_request_proto_info(nxt_task_t *task, nxt_http_request_t *r); +static nxt_bool_t nxt_http_request_is_bodyless(nxt_http_request_t *r); +static void nxt_http_request_drop_framing_fields(nxt_http_request_t *r); +static nxt_buf_t *nxt_http_request_body_drop(nxt_task_t *task, + nxt_http_request_t *r, nxt_buf_t *out); static void nxt_http_request_mem_buf_completion(nxt_task_t *task, void *obj, void *data); static void nxt_http_request_done(nxt_task_t *task, void *obj, void *data); @@ -692,6 +696,45 @@ nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } + /* + * RFC 9112 Sect. 6.3: a 1xx, 204 or 304 response, and any response to a + * HEAD request, never has a message body, no matter what the response + * headers say. Record that here, in the layer every response source + * (application, proxy, static, "return") passes through, so that the + * per-protocol header_send() can drop the framing and + * nxt_http_request_send() can drop the body buffers. Without this an + * application that writes a body on 204/304 has those bytes forwarded + * verbatim and unframed, which a downstream parser reads as the start of + * the next response. + */ + r->no_body = nxt_http_request_is_bodyless(r); + + /* + * RFC 9110 Sect. 8.6: a server must not send Content-Length in a 1xx or + * 204 response. A 304 keeps it -- there it describes the body the client + * already has -- and so does a HEAD response, where it describes the body + * the equivalent GET would return. + */ + if (r->no_body + && (r->status < NXT_HTTP_OK || r->status == NXT_HTTP_NO_CONTENT)) + { + if (r->resp.content_length != NULL) { + r->resp.content_length->skip = 1; + } + + r->resp.content_length_n = -1; + + /* + * r->resp.content_length only tracks the field the application sent. + * A Content-Length added by "response_headers" (nxt_http_set_headers.c) + * is a generic field, and an application-supplied Transfer-Encoding is + * one too; both are forbidden here and both would otherwise be + * serialized verbatim, desyncing the next response on a connection + * whose keep-alive this change preserves. Sweep the whole field store. + */ + nxt_http_request_drop_framing_fields(r); + } + /* * TODO: "Server", "Date", and "Content-Length" processing should be moved * to the last header filter. @@ -774,9 +817,144 @@ nxt_http_request_ws_frame_start(nxt_task_t *task, nxt_http_request_t *r, } +static nxt_bool_t +nxt_http_request_is_bodyless(nxt_http_request_t *r) +{ + /* + * A 101 upgrade is a 1xx status, but the bytes that follow its header are + * not a message body -- they are the upgraded protocol (WebSocket), and + * nxt_h1proto_websocket.c pushes them through nxt_http_request_send(). + * + * The status matters as much as the flag. websocket_handshake is set when + * the request headers are parsed (src/nxt_h1proto.c), long before the + * application chooses a status, and the h1 sender only treats the response + * as an upgrade when it is also 101 (src/nxt_h1proto.c). Testing the flag + * alone would leave an application that answers a WebSocket-upgrade + * request with 204 plus a body on the unframed path. + */ + if (r->websocket_handshake && r->status == NXT_HTTP_SWITCHING_PROTOCOLS) { + return 0; + } + + if (r->status >= NXT_HTTP_CONTINUE && r->status < NXT_HTTP_OK) { + return 1; + } + + if (r->status == NXT_HTTP_NO_CONTENT + || r->status == NXT_HTTP_NOT_MODIFIED) + { + return 1; + } + + return r->method != NULL && nxt_str_eq(r->method, "HEAD", 4); +} + + +/* + * Mark every Content-Length and Transfer-Encoding response field skipped. + * Only for 1xx and 204, where RFC 9110 Sect. 8.6 and RFC 9112 Sect. 6.1 forbid + * both: a 304 keeps Content-Length, and so does a response to HEAD. + */ + +static void +nxt_http_request_drop_framing_fields(nxt_http_request_t *r) +{ + nxt_http_field_t *field; + + nxt_http_fields_each(field, r->resp.inline_fields, r->resp.num_inline_fields, + r->resp.fields) + { + if (field->skip) { + continue; + } + + if ((field->name_length == nxt_length("Content-Length") + && nxt_strncasecmp(field->name, (u_char *) "Content-Length", + nxt_length("Content-Length")) == 0) + || (field->name_length == nxt_length("Transfer-Encoding") + && nxt_strncasecmp(field->name, (u_char *) "Transfer-Encoding", + nxt_length("Transfer-Encoding")) == 0)) + { + field->skip = 1; + } + + } nxt_http_fields_loop; +} + + +/* + * Strip the payload from a response that must not carry one, keeping the + * sync/last markers that drive request completion. Data-only buffers are + * unlinked and drained so their completion handlers run and the shared memory + * they hold is released; a buffer that also carries the "last" marker stays in + * the chain but is emptied in place, so the request still finishes normally. + */ + +static nxt_buf_t * +nxt_http_request_body_drop(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *out) +{ + size_t dropped; + nxt_buf_t *b, *next, **prev; + nxt_work_queue_t *wq; + + dropped = 0; + prev = &out; + wq = &task->thread->engine->fast_work_queue; + + for (b = out; b != NULL; b = next) { + next = b->next; + + if (nxt_buf_is_sync(b)) { + /* + * A sync buffer may be allocated with NXT_BUF_SYNC_SIZE, so it + * carries no payload and its mem.free/file fields must not even + * be read. Keep it as is. + */ + prev = &b->next; + continue; + } + + dropped += nxt_buf_used_size(b); + + if (nxt_buf_is_last(b)) { + /* Empty it in place; the "last" marker still ends the request. */ + b->mem.pos = b->mem.free; + + if (nxt_buf_is_file(b)) { + b->file_pos = b->file_end; + } + + prev = &b->next; + continue; + } + + *prev = next; + b->next = NULL; + + nxt_sendbuf_drain(task, wq, b); + } + + if (dropped != 0) { + nxt_debug(task, "http request body dropped on status %d: %uz bytes", + (int) r->status, dropped); + } + + return out; +} + + void nxt_http_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) { + if (r->no_body) { + out = nxt_http_request_body_drop(task, r, out); + + if (out == NULL) { + return; + } + } + if (nxt_fast_path(r->proto.any != NULL)) { nxt_http_proto[r->protocol].send(task, r, out); } diff --git a/test/php/no_body_status/index.php b/test/php/no_body_status/index.php new file mode 100644 index 000000000..ef2179920 --- /dev/null +++ b/test/php/no_body_status/index.php @@ -0,0 +1,15 @@ +websocket_handshake is set while the *request* headers are parsed, long + before the application picks a status, so the upgrade exemption has to test + the status too -- otherwise this request keeps the unframed path. + """ + + key = base64.b64encode(os.urandom(16)).decode() + + raw, sock = request_raw( + '/?status=204', + headers={ + 'Host': 'localhost', + 'Connection': 'Upgrade', + 'Upgrade': 'websocket', + 'Sec-WebSocket-Key': key, + 'Sec-WebSocket-Version': 13, + }, + ) + + try: + head, rest = split_response(raw) + + assert ' 204 ' in head.split('\r\n')[0], 'status line' + assert SMUGGLED not in raw, f'body forwarded on upgrade+204: {raw!r}' + assert rest == '', f'trailing bytes after header: {rest!r}' + + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'keep-alive after upgrade+204: status' + assert resp['body'] == SMUGGLED, 'keep-alive after upgrade+204: body' + + finally: + sock.close() + + +def test_no_body_status_app_transfer_encoding(): + """An application-sent Transfer-Encoding is a generic field; drop it.""" + + raw, sock = request_raw('/?status=204&te=1') + + try: + head, rest = split_response(raw) + + assert ' 204 ' in head.split('\r\n')[0], 'status line' + assert ( + 'Transfer-Encoding' not in head + ), f'Transfer-Encoding kept on 204: {head!r}' + assert SMUGGLED not in raw, f'body forwarded on 204: {raw!r}' + assert rest == '', f'trailing bytes after header: {rest!r}' + + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'keep-alive: status' + assert resp['body'] == SMUGGLED, 'keep-alive: body' + + finally: + sock.close() + + +def test_no_body_status_response_headers_content_length(): + """Framing headers added by "response_headers" must go too. + + They are added as generic fields, without setting r->resp.content_length, + so skipping that one pointer is not enough. Note the lowercase name: the + config validator rejects "Content-Length" here, but with a case-sensitive + memcmp (nxt_conf_vldt_response_header()), so "content-length" is accepted + and reaches the response. Transfer-Encoding is not guarded at all. + """ + + php_dir = f'{option.test_dir}/php' + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + { + # Scoped to the 204 so the follow-up 200 on the same + # connection is framed by the application alone. + "match": {"arguments": {"status": "204"}}, + "action": { + "pass": "applications/no_body_status", + "response_headers": { + "content-length": "10", + "Transfer-Encoding": "chunked", + }, + }, + }, + {"action": {"pass": "applications/no_body_status"}}, + ], + "applications": { + "no_body_status": { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "root": f'{php_dir}/no_body_status', + "working_directory": f'{php_dir}/no_body_status', + "index": "index.php", + } + }, + } + ), 'response_headers configuration' + + raw, sock = request_raw('/?status=204') + + try: + head, rest = split_response(raw) + + assert ' 204 ' in head.split('\r\n')[0], 'status line' + assert ( + 'content-length' not in head.lower() + ), f'response_headers Content-Length kept on 204: {head!r}' + assert ( + 'transfer-encoding' not in head.lower() + ), f'response_headers Transfer-Encoding kept on 204: {head!r}' + assert SMUGGLED not in raw, f'body forwarded on 204: {raw!r}' + assert rest == '', f'trailing bytes after header: {rest!r}' + + resp = client.get(sock=sock, url='/?status=200', headers=CLOSE) + + assert resp['status'] == 200, 'keep-alive: status' + assert resp['body'] == SMUGGLED, 'keep-alive: body' + + finally: + sock.close() From 7cda8d6d5830e1e680cda7f7d04f76396b48004d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 17:09:37 +0200 Subject: [PATCH 091/391] router: fail the start RPC when the prototype dies nxt_router_start_app_process_handler() (src/nxt_router.c:401) arms an RPC on the router's own port and sends START_PROCESS to the application's prototype (:469). It armed that registration with no peer, so the only thing that could ever retire it was a reply on its own stream: the worker's PROCESS_READY, forwarded by the prototype as NEW_PORT. A prototype that dies after forking the worker and before that worker's READY makes such a reply impossible. The router is told by REMOVE_PID, and that message carries no stream for a prototype which had itself reached READY (src/nxt_main_process.c:1200), so the RPC_ERROR branch in nxt_router_remove_pid_handler() (src/nxt_port.c:1126) does not fire; the sweep that runs instead, nxt_port_rpc_remove_peer() (src/nxt_port.c:1067), retires registrations by peer and found none. The orphaned worker is reparented to init, its READY write fails with EPIPE and it exits -- and the attempt's app->pending_processes increment is never given back. With "processes": {"spare": 0} that one stranded slot is the whole of max_pending_processes (src/nxt_router.c:2061), so nxt_router_app_can_start() (src/nxt_router.h) is false for the router's lifetime. Every later request joins ack_waiting_req, where nothing times it out, and no replacement prototype is ever asked for: the start handler is the only thing that asks for one (:428), and it is what can_start gates. The wedge is permanent, and a reload that leaves the application's configuration text unchanged reuses the wedged nxt_app_t rather than replacing it. Key the registration by the process the START_PROCESS was sent to. The REMOVE_PID sweep then runs nxt_router_app_port_error() (:5054) exactly once -- nxt_port_rpc_remove_peer() clears the peer, deletes the stream and frees the registration (src/nxt_port_rpc.c:486) -- which releases the slot through nxt_router_app_start_failed() and, when no process is left to answer, fails the ack-waiting requests with 503 instead of letting them hang. When a live worker remains they stay queued and it serves them. The next request re-enters the start handler and asks main for a fresh prototype. For a prototype start the peer is the main process, which is harmless: main's death is fatal to the daemon anyway, and a prototype that dies before its own READY is already reported by the stream-bearing REMOVE_PID. This mirrors the prefork site at :3355, which has keyed its registration by peer all along. src/test/nxt_router_proto_death_test.c drives the fixture through a start that succeeds and then the peer sweep the real REMOVE_PID performs, and requires the slot back and the application startable again; a repeated sweep must release nothing further. test_python_prototype_killed_mid_start() kills the prototype during a worker's import and requires the in-flight request answered, the status counter back at zero, and the next request served by a new prototype. Both this test and src/test/nxt_router_start_fail_soak_test.c need that predicate, which was nxt_inline in src/nxt_router.c and so private to the router's translation unit; each had kept a private copy of it, guarded only by a comment asking that they be kept in sync. That guard is worth nothing: adding a third term to the real predicate leaves both tests green, because neither was looking at it. Move nxt_router_app_can_start() to src/nxt_router.h, next to the struct nxt_app_s whose counters it reads and which both tests already include, and delete the copies. The same injected term now turns the run red. Fixes #269 Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + src/nxt_router.c | 31 ++- src/nxt_router.h | 16 ++ src/test/nxt_router_proto_death_test.c | 294 +++++++++++++++++++++ src/test/nxt_router_start_fail_soak_test.c | 17 +- src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/python/slow_start/wsgi.py | 16 ++ test/test_python_application.py | 170 ++++++++++++ 9 files changed, 526 insertions(+), 24 deletions(-) create mode 100644 src/test/nxt_router_proto_death_test.c create mode 100644 test/python/slow_start/wsgi.py diff --git a/auto/sources b/auto/sources index 529c0ba77..0f837b29d 100644 --- a/auto/sources +++ b/auto/sources @@ -182,6 +182,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_start_fail_test.c \ src/test/nxt_router_start_fail_soak_test.c \ src/test/nxt_router_proto_wedge_test.c \ + src/test/nxt_router_proto_death_test.c \ src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_port_change_file_test.c \ diff --git a/src/nxt_router.c b/src/nxt_router.c index 1728011a8..4179330f5 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -478,6 +478,29 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, app_joint_rpc->generation = app->generation; app_joint_rpc->proto = (b != NULL); + /* + * Key the registration by the process the START_PROCESS was sent to, so + * that its death retires the RPC. Without a peer the registration is + * only ever retired by a reply on its own stream: when the prototype + * dies between forking the worker and that worker's PROCESS_READY, the + * REMOVE_PID the router gets for the prototype carries no stream (it + * had already reached READY itself, see nxt_main_process.c) and + * nxt_port_rpc_remove_peer() finds nothing to fail, so the attempt's + * pending_processes slot is never given back. With spare 0 that alone + * exceeds max_pending_processes, nxt_router_app_can_start() stays false + * for good, and no later request can even ask for a new prototype -- + * only the start handler does that, and it is what can_start gates. + * + * With the peer set, nxt_port_rpc_remove_peer() runs + * nxt_router_app_port_error() exactly once (it clears ->peer, deletes + * the stream and frees the registration), which releases the slot and + * fails the ack-waiting requests when no process is left. For a + * prototype start the peer is the main process, which is harmless: its + * death is fatal anyway, and a prototype dying before READY is already + * reported by the stream-bearing REMOVE_PID. + */ + nxt_port_rpc_ex_set_peer(task, port, app_joint_rpc, dport->pid); + if (b != NULL) { app->proto_port_requests++; @@ -1220,14 +1243,6 @@ nxt_router_temp_conf(nxt_task_t *task) } -nxt_inline nxt_bool_t -nxt_router_app_can_start(nxt_app_t *app) -{ - return app->processes + app->pending_processes < app->max_processes - && app->pending_processes < app->max_pending_processes; -} - - nxt_inline nxt_bool_t nxt_router_app_need_start(nxt_app_t *app) { diff --git a/src/nxt_router.h b/src/nxt_router.h index d9d60d1db..ab3f6e54d 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -159,6 +159,22 @@ struct nxt_app_s { }; +/* + * The gate on every application start. It lives here rather than in + * src/nxt_router.c so that the regression tests which turn on it -- the ones + * for the pending_processes accounting, #214 and #269 -- can call the very + * predicate the router uses instead of keeping a copy that silently drifts + * from it. + */ + +nxt_inline nxt_bool_t +nxt_router_app_can_start(nxt_app_t *app) +{ + return app->processes + app->pending_processes < app->max_processes + && app->pending_processes < app->max_pending_processes; +} + + typedef struct { size_t max_frame_size; nxt_msec_t read_timeout; diff --git a/src/test/nxt_router_proto_death_test.c b/src/test/nxt_router_proto_death_test.c new file mode 100644 index 000000000..58649b25b --- /dev/null +++ b/src/test/nxt_router_proto_death_test.c @@ -0,0 +1,294 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for issue #269: a prototype that dies between forking a + * worker and that worker's PROCESS_READY strands the router's on-demand + * START_PROCESS registration. + * + * The router hears about the death as a REMOVE_PID for the prototype. That + * message carries no stream once the prototype itself has reached READY + * (src/nxt_main_process.c:1200), so the only thing left that can retire the + * armed RPC is nxt_port_rpc_remove_peer() (src/nxt_port.c:1067) -- and it + * retires registrations by peer. nxt_router_start_app_process_handler() + * used to arm its registration with no peer at all, so the sweep found + * nothing, the attempt's pending_processes slot was never given back, and + * with "processes: {spare: 0}" (max_pending_processes 1) + * nxt_router_app_can_start() stayed false for the router's lifetime: every + * later request parked in ack_waiting_req and no replacement prototype was + * ever asked for, because only the start handler asks for one and can_start + * is what gates it. + * + * This is the sibling of src/test/nxt_router_proto_wedge_test.c, which + * covers the prototype-start branch retired through nxt_port_rpc_close(). + * Here the application already has a prototype, so the handler takes the + * plain app-start branch, and the RPC is retired the way the real + * REMOVE_PID does it: by peer. Before the fix the sweep is a no-op and the + * checks below fail; after it the sweep runs nxt_router_app_port_error() + * exactly once, the slot comes back, and the application is startable again. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_proto_death_test_app; + + +nxt_int_t +nxt_router_proto_death_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *proto_port; + nxt_runtime_t *rt, *saved_rt; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test started"); + + ret = NXT_ERROR; + router_port = NULL; + proto_port = NULL; + joint = NULL; + app_mutex = 0; + app = &nxt_router_proto_death_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Only the members the reached code touches are set; see + * src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* + * nxt_port_rpc_register_handler_ex() asserts on a live pair[0]; the + * descriptor is never touched here, so borrow the nxt_port_fail_test() + * convention of a placeholder that is reset before the port is freed. + */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The prototype the START_PROCESS is written to, and whose pid the + * REMOVE_PID sweep is driven with. No queue and write_ready unset, so + * nxt_port_socket_write2() takes the nxt_port_msg_chk_insert() path and + * the message is simply queued: the write succeeds and the RPC stays + * armed, which is what this test needs. + * + * The port's pid is the running process, as everywhere in these + * fixtures; the sweep only ever compares it with the value the handler + * recorded as the registration's peer. + */ + + proto_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_PROTOTYPE); + if (nxt_slow_path(proto_port == NULL)) { + goto done; + } + + proto_port->pair[0] = -1; + proto_port->pair[1] = -1; + proto_port->socket.fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "proto-death-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + /* + * "processes: {spare: 0}" -- max_pending_processes is 1, so a single + * stranded slot is enough to wedge the application for good. + */ + app->max_processes = 8; + app->max_pending_processes = 1; + + /* A live prototype, so the handler takes the plain app-start branch. */ + app->proto_port = proto_port; + app->conf.length = 0; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->pending_processes != 1 || joint->use_count != 2 + || app->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: successful start left " + "pending %uD joint %uD app use %d (expected 1, 2 and 1)", + app->pending_processes, joint->use_count, + (int) app->use_count); + goto done; + } + + /* + * The prototype dies with the worker it forked not yet ready. This is + * exactly what nxt_router_remove_pid_handler() does with the streamless + * REMOVE_PID that reports it. + */ + + nxt_port_rpc_remove_peer(task, router_port, proto_port->pid); + + if (app->pending_processes != 0 || joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: prototype death left pending " + "%uD joint %uD (expected 0 and 1) -- the start RPC was " + "not retired", app->pending_processes, + joint->use_count); + goto done; + } + + /* + * The gate #269 wedges: with the slot back the application can ask for + * a replacement prototype on the next request. + */ + + app->processes = 0; + + if (!nxt_router_app_can_start(app)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: app cannot start after the " + "prototype's death, processes %uD pending %uD", + app->processes, app->pending_processes); + goto done; + } + + /* + * And the death is reported exactly once: a second sweep for the same + * pid, which a REMOVE_PID for an already-forgotten process would drive, + * must find nothing left to fail rather than release a second slot. + */ + + app->pending_processes = 1; + + nxt_port_rpc_remove_peer(task, router_port, proto_port->pid); + + if (app->pending_processes != 1 || joint->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: repeated sweep left pending " + "%uD joint %uD (expected 1 and 1)", + app->pending_processes, joint->use_count); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router proto death test passed"); + + ret = NXT_OK; + +done: + + if (proto_port != NULL) { + /* + * The start above was queued rather than written, and + * nxt_port_msg_chk_insert() takes a port reference for every queued + * message. Left alone the port never reaches nxt_port_mp_cleanup(), + * so its pool and messages leak and the assertions there never run + * -- which would quietly hollow out this test under a debug build. + */ + nxt_port_test_run_error_handler(task, proto_port); + + if (!nxt_queue_is_empty(&proto_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router proto death test: prototype port still " + "holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_use(task, proto_port, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_router_start_fail_soak_test.c b/src/test/nxt_router_start_fail_soak_test.c index 8ba7d88d9..8f2a1fd73 100644 --- a/src/test/nxt_router_start_fail_soak_test.c +++ b/src/test/nxt_router_start_fail_soak_test.c @@ -45,21 +45,6 @@ static nxt_app_t nxt_router_start_fail_soak_test_app; -/* - * Mirrors the nxt_inline nxt_router_app_can_start() predicate in - * src/nxt_router.c:1218. Being nxt_inline, the gate #214 wedges is private - * to the router's translation unit and cannot be called from here, so this - * copy must be kept in sync with it. - */ - -static nxt_bool_t -nxt_router_start_fail_soak_can_start(nxt_app_t *app) -{ - return app->processes + app->pending_processes < app->max_processes - && app->pending_processes < app->max_pending_processes; -} - - nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr) { @@ -255,7 +240,7 @@ nxt_router_start_fail_soak_test(nxt_thread_t *thr) app->conf.length = 0; app->proto_port = dport; - if (!nxt_router_start_fail_soak_can_start(app)) { + if (!nxt_router_app_can_start(app)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, "router start fail soak test: app cannot start after " "%d failures, processes %uD pending %uD", diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index a10a8127a..d494ca9dd 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -213,6 +213,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_proto_death_test(thr) != NXT_OK) { + return 1; + } + if (nxt_router_remove_pid_soak_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 40878c749..5f9e668d0 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -77,6 +77,7 @@ nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); +nxt_int_t nxt_router_proto_death_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); diff --git a/test/python/slow_start/wsgi.py b/test/python/slow_start/wsgi.py new file mode 100644 index 000000000..be17d09d4 --- /dev/null +++ b/test/python/slow_start/wsgi.py @@ -0,0 +1,16 @@ +import os +import time + +# Widen the window between the worker's PROCESS_CREATED and its PROCESS_READY, +# which is what test_python_prototype_killed_mid_start() kills the prototype +# inside of. The delay is at import time, before `application` is even +# defined: nxt_python_start() imports the module and only then sends READY +# (src/python/nxt_python.c), so a sleep here lands squarely in that window, +# while a sleep inside application() would not -- the process is ready long +# before a request arrives. +time.sleep(float(os.environ.get('UNIT_SLOW_START', '4'))) + + +def application(environ, start_response): + start_response('200 OK', [('Content-Length', '0')]) + return [] diff --git a/test/test_python_application.py b/test/test_python_application.py index 6563efd2d..f4a066c64 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -4,6 +4,7 @@ import pwd import re import subprocess +import threading import time import venv @@ -1052,3 +1053,172 @@ def test_python_application_upload_multiple_files(): ) assert resp['status'] == 200, 'multiple files status' assert resp['body'] == 'first.txtfirst data', 'multiple files body' + + +def _ps_rows(): + out = subprocess.check_output( + ['ps', 'ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'args='] + ).decode() + + return [ + parts + for parts in (line.split(None, 2) for line in out.splitlines()) + if len(parts) == 3 + ] + + +def _proto_pid(name): + """The pid of the application's prototype process, or None. + + `unit: "" prototype` is the title nxt_main_process.c gives it, and + unlike a worker's title it survives for every runtime -- the prototype + never execve()s. Matched on the whole ps line for the same reason + test_app_lifecycle.py does. + """ + + marker = f'unit: "{name}" prototype' + + for pid, _, args in _ps_rows(): + if marker in args: + return pid + + return None + + +def _child_pids(ppid): + return [pid for pid, parent, _ in _ps_rows() if parent == ppid] + + +def test_python_prototype_killed_mid_start(skip_alert, findall): + """Issue #269: a prototype killed between forking a worker and that + worker's PROCESS_READY must not strand the router's START_PROCESS. + + The router takes an app->pending_processes slot before posting + nxt_router_start_app_process_handler(), which arms an RPC on the router + port and sends START_PROCESS to the prototype. The reply that would + retire it -- the worker's READY, forwarded by the prototype -- can never + arrive once the prototype is dead, and the REMOVE_PID that reports the + death carries no stream for a prototype that had itself reached READY. + Only a peer-keyed registration is retired by that message. Without one + the slot is never given back; with "spare": 0 that is the whole of + max_pending_processes, so nxt_router_app_can_start() is false for good, + every later request parks in ack_waiting_req with no timeout, and no + replacement prototype is ever requested -- the start handler is the only + thing that asks, and it is what can_start gates. The wedge is permanent + and survives a reload that leaves the app's config text unchanged. + + So the assertions are: the in-flight request is answered rather than + hung, /status shows the slot back at zero, and the next request gets a + fresh prototype and a 200. On the unfixed router the first of those + never returns, which is why every wait here is bounded. + + The app module sleeps in its *import* (test/python/slow_start/wsgi.py), + which is what widens the PROCESS_CREATED -> READY window enough to aim + at. A kill that lands outside the window makes the test skip rather + than assert something it did not drive. + """ + + client.load( + 'slow_start', + processes={'max': 2, 'spare': 0, 'idle_timeout': 1}, + environment={'UNIT_SLOW_START': '4'}, + ) + + resp = [] + + def _request(): + try: + # Bounded: on the unfixed router this request is never answered, + # and a thread stuck in recv() would outlive the test. + resp.append(client.get(read_timeout=20)['status']) + + except Exception as exc: # noqa: BLE001 - reported as a failure below + resp.append(repr(exc)) + + thread = threading.Thread(target=_request) + thread.start() + + try: + # The prototype appears as soon as main forks it; the worker it then + # forks spends the next 4s importing the module. + deadline = time.time() + 10 + pid = None + + while time.time() < deadline: + pid = _proto_pid('slow_start') + + if pid is not None: + break + + time.sleep(0.1) + + assert pid is not None, 'prototype never started' + + # Wait for the worker to exist: killing the prototype before it has + # forked would carry the original start stream and let an unfixed + # router pass this test by the other route. Then a short pause puts + # the worker mid-import: past PROCESS_CREATED, well before READY. + workers = [] + for _ in range(100): + workers = _child_pids(pid) + if workers: + break + time.sleep(0.05) + + assert workers, 'the prototype never forked a worker' + + time.sleep(0.3) + + # The worker is still importing the module. Its own teardown noise + # is scoped to its pid below rather than skipped by shape, so an + # alert from any other process still fails the test. + + subprocess.call(['kill', '-9', pid]) + skip_alert(fr'process {pid} exited on signal 9') + + for worker in workers: + # Orphaned by the kill, the worker finishes its import and then + # writes READY into the dead prototype's socket, which is EPIPE, + # and unwinds closing descriptors it inherited from the + # prototype -- which the kill already closed. All of that is + # the pre-existing consequence of killing a prototype, not + # something this test's subject produces; it happens on the + # unfixed router too. + skip_alert(fr'\b{worker}#{worker} ') + + # The kill retires the start RPC, which releases the slot and -- no + # process being left -- fails the waiting request instead of letting + # it sit forever. + thread.join(timeout=25) + + finally: + if thread.is_alive(): + # Do not leave a thread blocked in recv() behind; the assertion + # below reports the wedge. + thread.join(timeout=10) + + assert not thread.is_alive(), ( + 'the in-flight request never completed: the start RPC was stranded' + ) + assert resp == [503], f'in-flight request: {resp}' + + # The slot is back, so the application is startable again. + deadline = time.time() + 10 + starting = None + + while time.time() < deadline: + starting = client.conf_get('/status')['applications']['slow_start'][ + 'processes' + ]['starting'] + + if starting == 0: + break + + time.sleep(0.2) + + assert starting == 0, f'processes.starting stuck at {starting}' + + # And the next request gets a brand new prototype and is served. + assert client.get(read_timeout=30)['status'] == 200, 'recovered' + assert _proto_pid('slow_start') not in (None, pid), 'fresh prototype' + assert findall(fr'process {pid} exited on signal 9'), 'kill logged' From af4f04458b91885edfaabf4e0567f199b1e0ffec Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 17:13:09 +0200 Subject: [PATCH 092/391] proxy: complete a bodyless upstream response on its headers nxt_h1p_peer_header_read_done() (src/nxt_h1proto.c:2854) armed the upstream body framing from the upstream headers alone: a Content-Length became h1p->remainder, a "Transfer-Encoding: chunked" armed h1p->chunked_parse. Neither the request method nor the response status was consulted. RFC 9112 Sect. 6.3 rule 1 says those headers do not frame anything here: a response to HEAD, and any 204 or 304 response, ends at the first empty line after the header fields whatever Content-Length or Transfer-Encoding say. So the upstream sends the header block and nothing more -- and, because nxt_h1p_peer_header_send() writes an unconditional "Connection: close" to every upstream (src/nxt_h1proto.c:2541), it then closes. nxt_h1p_peer_closed() (src/nxt_h1proto.c:3172) reads that close as a cut body: h1p->remainder is still the advertised length, or chunked_parse.last is still clear, so it sets r->truncated and r->inconsistent. nxt_h1p_request_close() (src/nxt_h1proto.c:1929) then does "h1p->keepalive &= !r->inconsistent", and the client connection is torn down after a response that was never short. Any request the client had already pipelined into the same socket is lost -- and a proxied HEAD is the ordinary case here, not a corner one. Decide it where the framing is chosen. #283 put the same rule in nxt_http_request_header_send() for the response Unit sends downstream; this is its upstream half, and it shares that helper's status list rather than restating it. nxt_http_request_is_bodyless() now takes the status as a parameter and is no longer static: the peer reader has to ask the question while the upstream status still lives in peer->status, one step before nxt_http_proxy_header_read() (src/nxt_http_proxy.c:256) copies it into r->status. The peer reader uses a narrower predicate, nxt_http_request_is_bodyless_final(): HEAD, 204 and 304, but not 1xx. "Carries no body" and "ends the exchange" are the same question downstream and different questions here, because a 1xx is an interim response. Upstream 1xx handling does not exist to build on: nxt_h1p_peer_header_parse() (src/nxt_h1proto.c:2941) reads a status line only while peer->status is still NXT_HTTP_UNSET, so a 1xx is already taken as the response and whatever follows is relayed as its body. That is pre-existing and out of scope; treating the 1xx header as the end of the exchange would discard a final response that may already be sitting in the same read buffer. The 101 upgrade path is untouched either way. For a bodyless upstream response the peer reader now neither arms the chunked parser nor sets a remainder, and completes the response right there the way nxt_h1p_peer_body_process() completes a body that reached its declared length: peer->body = nxt_http_buf_last(r), peer->closed = 1, then the ready handler. nxt_http_proxy_send_body() relays that last buffer, closes the upstream connection and releases the request pool. The header buffer is not forwarded: bytes an upstream put after the header of a bodyless response are not a body. It is not freed either, and it cannot simply be dropped -- both would be bugs, in opposite directions: - Freeing it dangles the response fields. peer->fields point their name/value into that buffer (nxt_http_parse_fields, src/nxt_http_parse.c:813) and nxt_http_proxy_header_read() shallow-copies those field structs into r->resp, where $response_header_* and "match" conditions read them until the request is logged and closed. - Dropping it strands the request pool. nxt_http_proxy_buf_mem_alloc() retains r->mem_pool for every upstream read buffer, and that retain is released by a buffer completion -- which a buffer that is never relayed never gets. nxt_http_proxy_buf_mem_hold() (src/nxt_http_proxy.c) does both at once: nxt_mp_cleanup() returns the buffer to the engine when the pool is destroyed, and the alloc-time retain is dropped so the pool can reach zero at all. That drop path is not new here. nxt_h1p_peer_header_read_done() has always dropped the buffer when the header read carried no body bytes with it -- which is every 204 and 304, every response to HEAD, and any upstream that writes its header and body in separate segments. Measured with a throwaway build that forces nxt_debug on and counts pools reaching a zero retain, over 200 proxied requests each: upstream shape master this commit 204, header block only 400 600 HEAD, header block only 400 600 HEAD, header + 10 trailing bytes 600 600 200, header + body in one read 600 600 Three pools reach zero per request when nothing is stranded; 400 is two, the missing one being the request pool. So the same hold is applied to that pre-existing path as well, and both columns now read 600 throughout. Client-side framing is unchanged and stays correct. A HEAD response keeps the upstream's Content-Length (RFC 9110 Sect. 9.3.2) because #283 only drops the framing fields for 1xx and 204; a 304 keeps it too. Nothing follows the header block, and rule 1 lets the client end the message there, so keep-alive is safe without a Content-Length -- which is what a HEAD of a chunked upstream resource produces, since the hop-by-hop Transfer-Encoding is not relayed. The duplicate-framing guard is untouched: an upstream that sends both chunked and Content-Length is still a 502, and it is still checked before this. test/test_proxy_head.py pipelines two requests in a single write -- the bodyless one and a plain GET -- against a raw upstream that answers with the header block alone and closes. The second response arriving is the regression assertion. Against the parent commit every bodyless case fails there, with only the first response on the wire and no "Connection: close" in it: the client is given no signal that its pipelined request will never be answered. Its upstream port is registered in test/fake_upstream/README.md, the registry for the 79xx block. One case has the upstream write its header and 10 junk bytes in a single send, so the bytes land in the same read as the header; that is the exit where the buffer used to reach nxt_h1p_peer_body_process() and be released by its completion. Another reads the relayed Content-Length back through $response_header_content_length in the access log, covering the field pointers on this new path. Neither asserts on the pool: a stranded request pool is invisible from the client side, and a stock build exposes no counter for it -- nxt_debug is never assigned, so even a --debug build emits no "mp ... release" lines. Hence the throwaway instrumented build above. For the same reason the field pointers are covered but not policed: freeing that buffer early leaves them dangling without reliably corrupting them, and 4800 concurrent proxied HEADs produced no corrupt record. Refs #283 Co-Authored-By: Claude Opus 5 --- src/nxt_h1proto.c | 75 ++++++++- src/nxt_http.h | 6 + src/nxt_http_proxy.c | 70 +++++++++ src/nxt_http_request.c | 49 ++++-- test/fake_upstream/README.md | 3 +- test/test_proxy_head.py | 284 +++++++++++++++++++++++++++++++++++ 6 files changed, 470 insertions(+), 17 deletions(-) create mode 100644 test/test_proxy_head.py diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 1900b4df8..748474694 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -2851,12 +2851,67 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) h1p = peer->proto.h1; - if (h1p->chunked) { - if (r->resp.content_length != NULL) { - peer->status = NXT_HTTP_BAD_GATEWAY; + if (h1p->chunked && r->resp.content_length != NULL) { + peer->status = NXT_HTTP_BAD_GATEWAY; + break; + } + + /* + * RFC 9112 Sect. 6.3: a response to HEAD, and any 204 or 304 response, + * is terminated by the first empty line after the header fields no + * matter what Content-Length or Transfer-Encoding say. Such a response + * is complete right here, so neither arm the chunked parser nor set a + * remainder from a Content-Length that describes a body the upstream + * will never send. + * + * Without this the upstream -- which nxt_h1p_peer_header_send() always + * asks to "Connection: close" -- closes with h1p->remainder still at + * the advertised length, or with chunked_parse.last still clear; + * nxt_h1p_peer_closed() then reads that as a truncated body and sets + * r->truncated and r->inconsistent, and nxt_h1p_request_close() drops + * the client keep-alive over a response that was never short. Any + * pipelined request already in the client's socket buffer is lost. + * + * Complete the response the way a body that reaches its declared + * length completes it in nxt_h1p_peer_body_process(): hand the + * request's last buffer to the ready handler and mark the peer closed, + * so nxt_http_proxy_send_body() closes the upstream connection and + * releases the request pool. + * + * The predicate is the "final response" one, not the full RFC list: + * a 1xx from an upstream is an interim response, and nothing here + * continues the exchange past it -- nxt_h1p_peer_header_parse() only + * reads a status line while peer->status is still NXT_HTTP_UNSET, so + * the 1xx is taken as the response and whatever follows is relayed as + * its body. That is pre-existing and out of scope; ending the + * exchange on the 1xx header here would discard a final response that + * may already be sitting in this very buffer. + * + * "b" is not forwarded: bytes an upstream put after the header of a + * bodyless response are not a body. It is handed to + * nxt_http_proxy_buf_mem_hold() rather than freed, because the + * response fields point their name/value into it and are read until + * the request is logged and closed; see the comment there. + */ + if (nxt_http_request_is_bodyless_final(r, peer->status)) { + h1p->chunked = 0; + h1p->remainder = 0; + + if (nxt_slow_path(nxt_http_proxy_buf_mem_hold(task, r, b) + != NXT_OK)) + { + peer->status = NXT_HTTP_INTERNAL_SERVER_ERROR; break; } + peer->body = nxt_http_buf_last(r); + peer->closed = 1; + + r->state->ready_handler(task, r, peer); + return; + } + + if (h1p->chunked) { h1p->chunked_parse.mem_pool = c->mem_pool; } else if (r->resp.content_length_n > 0) { @@ -2868,6 +2923,20 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) return; } + /* + * No body bytes arrived with the header, so nothing will relay "b" and + * nothing will run its completion handler. Dropping it here -- which + * is what this path did -- stranded both the buffer and the + * r->mem_pool retain nxt_http_proxy_buf_mem_alloc() took, so the whole + * request pool was never destroyed. Measured on the parent commit: + * two pools reach a zero retain per request on this path against three + * on the path where the body shares the header's read. + */ + if (nxt_slow_path(nxt_http_proxy_buf_mem_hold(task, r, b) != NXT_OK)) { + peer->status = NXT_HTTP_INTERNAL_SERVER_ERROR; + break; + } + r->state->ready_handler(task, r, peer); return; diff --git a/src/nxt_http.h b/src/nxt_http.h index b3974d8a9..49affb288 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -431,6 +431,10 @@ void nxt_http_request_ws_frame_start(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *ws_frame); void nxt_http_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out); +nxt_bool_t nxt_http_request_is_bodyless(nxt_http_request_t *r, + nxt_http_status_t status); +nxt_bool_t nxt_http_request_is_bodyless_final(nxt_http_request_t *r, + nxt_http_status_t status); nxt_buf_t *nxt_http_buf_mem(nxt_task_t *task, nxt_http_request_t *r, size_t size); nxt_buf_t *nxt_http_buf_last(nxt_http_request_t *r); @@ -519,6 +523,8 @@ nxt_int_t nxt_http_proxy_skip(void *ctx, nxt_http_field_t *field, uintptr_t data); nxt_buf_t *nxt_http_proxy_buf_mem_alloc(nxt_task_t *task, nxt_http_request_t *r, size_t size); +nxt_int_t nxt_http_proxy_buf_mem_hold(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *b); void nxt_http_proxy_buf_mem_free(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *b); diff --git a/src/nxt_http_proxy.c b/src/nxt_http_proxy.c index f7874ec47..db40f2a74 100644 --- a/src/nxt_http_proxy.c +++ b/src/nxt_http_proxy.c @@ -374,6 +374,76 @@ nxt_http_proxy_buf_mem_free(nxt_task_t *task, nxt_http_request_t *r, } +static void +nxt_http_proxy_buf_mem_cleanup(nxt_task_t *task, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_event_engine_t *engine; + + b = obj; + engine = data; + + /* + * Runs from nxt_mp_destroy(), which may be reached from a different task + * than the one current when the buffer was held, so "task" is not touched + * and the engine is carried in "data" instead. + */ + + nxt_event_engine_buf_mem_free(engine, b); +} + + +/* + * Keep an upstream read buffer alive for the rest of the request without + * holding the request pool hostage. + * + * A buffer that is relayed downstream needs none of this: its completion + * handler frees it and drops the retain. One that is *not* relayed does -- + * an upstream header block with no body bytes behind it, and the header of a + * response that RFC 9112 Sect. 6.3 gives no body at all. Two things then have + * to hold at once: + * + * - It must outlive the response. peer->fields point their name/value into + * this buffer (nxt_http_parse_fields) and nxt_http_proxy_header_read() + * shallow-copies those field structs into r->resp, where + * $response_header_* and "match" conditions read them until the request is + * logged and closed. Returning it to the engine cache when the response + * completes would leave them pointing at reusable memory. + * + * - It must not keep the retain nxt_http_proxy_buf_mem_alloc() took. That + * retain is dropped by a buffer completion, and a buffer that is never + * relayed has no completion to run -- the pool would never reach zero and + * the entire request pool would be stranded. + * + * A pool cleanup satisfies both: nxt_mp_destroy() runs it before freeing the + * pool's own blocks, so the field bytes stay valid for exactly as long as + * anything can read them, and not one request longer. + */ + +nxt_int_t +nxt_http_proxy_buf_mem_hold(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *b) +{ + nxt_int_t ret; + + ret = nxt_mp_cleanup(r->mem_pool, nxt_http_proxy_buf_mem_cleanup, task, b, + task->thread->engine); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + /* + * The request itself holds a retain until nxt_http_request_close_handler(), + * so this cannot destroy the pool here -- which it must not, the cleanup + * just registered lives in it. + */ + + nxt_mp_release(r->mem_pool); + + return NXT_OK; +} + + static void nxt_http_proxy_error(nxt_task_t *task, void *obj, void *data) { diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 91287ea73..ae5ed8d92 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -22,7 +22,6 @@ static void nxt_http_request_forward_protocol(nxt_http_request_t *r, static void nxt_http_request_ready(nxt_task_t *task, void *obj, void *data); static void nxt_http_request_proto_info(nxt_task_t *task, nxt_http_request_t *r); -static nxt_bool_t nxt_http_request_is_bodyless(nxt_http_request_t *r); static void nxt_http_request_drop_framing_fields(nxt_http_request_t *r); static nxt_buf_t *nxt_http_request_body_drop(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out); @@ -707,7 +706,7 @@ nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, * verbatim and unframed, which a downstream parser reads as the start of * the next response. */ - r->no_body = nxt_http_request_is_bodyless(r); + r->no_body = nxt_http_request_is_bodyless(r, r->status); /* * RFC 9110 Sect. 8.6: a server must not send Content-Length in a 1xx or @@ -817,8 +816,38 @@ nxt_http_request_ws_frame_start(nxt_task_t *task, nxt_http_request_t *r, } -static nxt_bool_t -nxt_http_request_is_bodyless(nxt_http_request_t *r) +/* + * A *final* response that RFC 9112 Sect. 6.3 gives no message body: 204, 304, + * or any response to HEAD. 1xx is excluded on purpose. A 1xx is an interim + * response -- the exchange continues with a final response after it -- so + * "carries no body" and "ends the exchange" are different questions for it, + * and a caller that needs the second one must not be answered with the first. + * + * The status is a parameter rather than a read of r->status because the proxy + * has to answer this question before r->status exists. The h1 peer reader + * decides how to frame the upstream body while the upstream status still lives + * in peer->status; nxt_http_proxy_header_read() copies it into r->status only + * one step later. + */ + +nxt_bool_t +nxt_http_request_is_bodyless_final(nxt_http_request_t *r, + nxt_http_status_t status) +{ + if (status < NXT_HTTP_OK) { + return 0; + } + + if (status == NXT_HTTP_NO_CONTENT || status == NXT_HTTP_NOT_MODIFIED) { + return 1; + } + + return r->method != NULL && nxt_str_eq(r->method, "HEAD", 4); +} + + +nxt_bool_t +nxt_http_request_is_bodyless(nxt_http_request_t *r, nxt_http_status_t status) { /* * A 101 upgrade is a 1xx status, but the bytes that follow its header are @@ -832,21 +861,15 @@ nxt_http_request_is_bodyless(nxt_http_request_t *r) * alone would leave an application that answers a WebSocket-upgrade * request with 204 plus a body on the unframed path. */ - if (r->websocket_handshake && r->status == NXT_HTTP_SWITCHING_PROTOCOLS) { + if (r->websocket_handshake && status == NXT_HTTP_SWITCHING_PROTOCOLS) { return 0; } - if (r->status >= NXT_HTTP_CONTINUE && r->status < NXT_HTTP_OK) { + if (status >= NXT_HTTP_CONTINUE && status < NXT_HTTP_OK) { return 1; } - if (r->status == NXT_HTTP_NO_CONTENT - || r->status == NXT_HTTP_NOT_MODIFIED) - { - return 1; - } - - return r->method != NULL && nxt_str_eq(r->method, "HEAD", 4); + return nxt_http_request_is_bodyless_final(r, status); } diff --git a/test/fake_upstream/README.md b/test/fake_upstream/README.md index 635d8b9f5..db3ab7c50 100644 --- a/test/fake_upstream/README.md +++ b/test/fake_upstream/README.md @@ -76,6 +76,7 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: | Port | Token | Mode (CLI) | Rust handler | pytest | |------|-------|-----------|--------------|--------| +| 7978 | — | — (plain-Python upstream in the test file) | `test_proxy_head.py` (bodyless upstream response: header block only, then close — not a `fake_upstream` mode, reserved here so the 79xx block stays in one registry) | | 7979 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_keepalive_disabled` (duplicate upstream Content-Length disables downstream keepalive for a keep-alive client; complete body keeps its terminal chunk) — out of the 7983–7999 block below, which is exhausted; 7980–7982 are `fake_otlp`, so 7974–7979 is the free gap | | 7983 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_raw` (zero raw `Content-Length` occurrences on the wire, complete re-framed chunked body) | | 7984 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl` (duplicate upstream Content-Length → neither forwarded, body re-framed, #113) | @@ -96,7 +97,7 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: > **7999 is not available here.** `test/test_proxy.py` and `test/test_proxy_chunked.py` > already bind `SERVER_PORT = 7999` for their own upstream, so this registry ends at -> 7998. New `fake_upstream` slots go to the 7974–7979 gap (7980–7982 are `fake_otlp`). +> 7998. New `fake_upstream` slots go to the 7974–7977 gap (7978–7979 are taken above, 7980–7982 are `fake_otlp`). A test pins its port as a module constant referencing this table: diff --git a/test/test_proxy_head.py b/test/test_proxy_head.py new file mode 100644 index 000000000..50db5f2c8 --- /dev/null +++ b/test/test_proxy_head.py @@ -0,0 +1,284 @@ +"""Proxied responses that RFC 9112 Sect. 6.3 defines as having no body. + +nxt_h1p_peer_header_read_done() used to arm the upstream body framing from the +upstream headers alone: Content-Length became h1p->remainder, Transfer-Encoding +armed the chunked parser. A response to HEAD, and any 204 or 304 response, +carries no body no matter what those headers say, so the upstream -- which +nxt_h1p_peer_header_send() always asks to "Connection: close" -- closed with the +remainder still outstanding. nxt_h1p_peer_closed() read that as a truncated +body and set r->truncated/r->inconsistent, and nxt_h1p_request_close() then +dropped the client keep-alive, losing any pipelined follow-up request. + +Each case pipelines two requests in a single write: the bodyless one, then a +plain GET. The second response arriving at all is the regression assertion. + +1xx is out of scope here: an upstream 1xx is an interim response, and nothing +in the peer reader continues the exchange past it (that is pre-existing, see +nxt_h1p_peer_header_parse), so it is left on the path it already had. + +These cases also cover the two exits an unrelayed upstream header buffer can +take, but they cannot assert that it is released: a stranded request pool is +invisible from the client side, and a stock build exposes no pool counter -- +nxt_debug is never assigned, so even a --debug build emits no "mp ... release" +lines. That was measured instead with a throwaway build that forces nxt_debug +on and counts pools reaching a zero retain per request; see the commit message. + +See freeunitorg/freeunit#283 for the downstream half of the same rule. +""" + +import select +import socket + +import pytest + +from conftest import run_process +from unit.applications.proto import ApplicationProto +from unit.option import option +from unit.utils import waitforsocket + +client = ApplicationProto() + +# Reserved in test/fake_upstream/README.md's port registry; 7980-7982 +# belong to test/fake_otlp. +UPSTREAM_PORT = 7978 + +# What the upstream answers, keyed by the request target Unit forwards. +# Every one of these closes the connection right after writing what is here: +# none of them ever sends the body its own headers advertise. +UPSTREAM_RESPONSES = { + '/cl10': 'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n', + '/chunked': 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n', + '/204': 'HTTP/1.1 204 No Content\r\nContent-Length: 10\r\n\r\n', + '/304': 'HTTP/1.1 304 Not Modified\r\nContent-Length: 10\r\n\r\n', + # Header block and 10 junk bytes in a single write, so the bytes land in + # the same read as the header. A response to HEAD has no body, so they + # must be dropped rather than relayed -- and the buffer holding them takes + # a different exit from nxt_h1p_peer_header_read_done() than the cases + # above, which is where an earlier revision of this change stranded the + # request pool. + '/cl10junk': ( + 'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nJUNKJUNKJU' + ), + # The trailing request of every pipeline; this one does have a body. + '/ok': 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK', +} + + +def run_server(server_port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.bind(('127.0.0.1', server_port)) + sock.listen(10) + + def recv_request(conn): + data = b'' + + while b'\r\n\r\n' not in data: + if not select.select([conn], [], [], 5)[0]: + break + + part = conn.recv(4096) + if not part: + break + + data += part + + return data.decode('utf-8', errors='ignore') + + while True: + conn, _ = sock.accept() + + request = recv_request(conn) + target = request.split(' ')[1] if ' ' in request else '' + + conn.sendall( + UPSTREAM_RESPONSES.get( + target, 'HTTP/1.1 500 Internal Server Error\r\n\r\n' + ).encode() + ) + conn.close() + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + run_process(run_server, UPSTREAM_PORT) + waitforsocket(UPSTREAM_PORT) + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"action": {"proxy": f'http://127.0.0.1:{UPSTREAM_PORT}'}} + ], + } + ), 'bodyless proxy configuration' + + +def pipeline(method, target): + """Send " " and a GET /ok back to back, read to EOF. + + Both requests leave in one write, so the second is already sitting in the + listener's read buffer when the first response is generated: it can only be + answered if the connection survives that response. + """ + + request = ( + f'{method} {target} HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + 'GET /ok HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + '\r\n' + ) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.settimeout(10) + + try: + sock.connect(('127.0.0.1', 8080)) + sock.sendall(request.encode()) + + data = b'' + while True: + try: + part = sock.recv(4096) + except socket.timeout: + break + + if not part: + break + + data += part + + finally: + sock.close() + + return data.decode('utf-8', errors='ignore') + + +def check_pipeline(raw, status): + """Assert the exact wire shape: bodyless header, then the whole GET /ok. + + Splitting on the header terminator pins both halves at once -- a body + leaking out of the first response, a chunked terminator appended to it, or + a missing second response all change the number of parts or their content. + """ + + parts = raw.split('\r\n\r\n') + + assert len(parts) == 3, f'expected two responses, got {raw!r}' + + first, second, body = parts + + assert first.startswith( + f'HTTP/1.1 {status}' + ), f'first response status line: {first!r}' + assert ( + 'close' not in first.lower() + ), f'keep-alive dropped on a bodyless response: {first!r}' + assert ( + 'transfer-encoding' not in first.lower() + ), f'chunked framing on a bodyless response: {first!r}' + + assert second.startswith( + 'HTTP/1.1 200' + ), f'pipelined request lost: {second!r}' + assert body == 'OK', f'pipelined response body: {body!r}' + + return first + + +def test_proxy_head_content_length(): + """HEAD: the upstream advertises a body and sends none, then closes.""" + + first = check_pipeline(pipeline('HEAD', '/cl10'), 200) + + # RFC 9110 Sect. 9.3.2: a HEAD response keeps the length the equivalent + # GET would report. + assert 'Content-Length: 10' in first, f'HEAD keeps Content-Length: {first!r}' + + +def test_proxy_head_chunked(): + """HEAD: the upstream announces chunked and sends no terminating chunk.""" + + check_pipeline(pipeline('HEAD', '/chunked'), 200) + + +def test_proxy_head_trailing_bytes(): + """HEAD: the upstream writes the header and 10 junk bytes together. + + A response to HEAD has no body, so those bytes are not one: they must not + reach the client and must not be counted as a body. They also put the + header buffer on the branch that used to hand it to + nxt_h1p_peer_body_process(), whose completion released it -- so this is the + case where simply returning early strands the buffer and, with it, the + request pool. That leak is not observable from here (see the module + docstring); what is checked is that the bytes are dropped and the + connection survives. + """ + + first = check_pipeline(pipeline('HEAD', '/cl10junk'), 200) + + assert 'Content-Length: 10' in first, f'HEAD keeps Content-Length: {first!r}' + + +def test_proxy_get_204(): + """204: Content-Length is advertised, no body follows, upstream closes.""" + + first = check_pipeline(pipeline('GET', '/204'), 204) + + # RFC 9110 Sect. 8.6 forbids Content-Length on a 204; #283 drops it. + assert ( + 'content-length' not in first.lower() + ), f'Content-Length kept on 204: {first!r}' + + +def test_proxy_get_304(): + """304: Content-Length describes the cached body and is kept.""" + + first = check_pipeline(pipeline('GET', '/304'), 304) + + assert 'Content-Length: 10' in first, f'304 keeps Content-Length: {first!r}' + + +def test_proxy_head_response_header_variable(wait_for_record): + """The relayed header bytes stay readable after the response completes. + + peer->fields point their name/value into the buffer the upstream header was + parsed from, and nxt_http_proxy_header_read() shallow-copies those field + structs into r->resp; $response_header_* reads them at access-log time. + This exercises that path on the early-complete branch, where the peer + connection is already closed by the time the record is written. + + It is coverage, not a use-after-free detector: returning that buffer to the + engine cache early leaves the pointers dangling but does not reliably + corrupt them, so this passes either way. The buffer must simply not be + freed there -- which is also what the normal path does. + """ + + assert 'success' in client.conf( + { + 'path': f'{option.temp_dir}/access.log', + 'format': '$request_line "$response_header_content_length"\n', + }, + 'access_log', + ), 'access_log format' + + check_pipeline(pipeline('HEAD', '/cl10'), 200) + + assert ( + wait_for_record(r'HEAD /cl10 HTTP/1.1 "10"', 'access.log') is not None + ), 'Content-Length readable from the access log after the response' + + +def test_proxy_get_body_still_relayed(): + """Control: a status that does define a body still gets one.""" + + raw = pipeline('GET', '/ok') + + assert raw.count('HTTP/1.1 200') == 2, f'expected two responses: {raw!r}' + assert raw.count('\r\n\r\nOK') == 2, f'both bodies relayed: {raw!r}' From 940ea082fb21528729bb305ad81bca0710e02c5a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 18:19:51 +0200 Subject: [PATCH 093/391] test(python): make the prototype-kill test skip outside its window Review follow-up to #288. test_python_prototype_killed_mid_start() promised in its docstring that "a kill that lands outside the window makes the test skip", and nothing in it called pytest.skip(). What it did instead was assert: a runner slow enough to let the worker reach READY before the kill served the request with 200 and failed at `assert resp == [503]`, which points at the fix rather than at the runner. Give it the promised guard. Immediately before the kill, /status has to still report one pending start -- "starting" is app->pending_processes, which the router gives back only when the worker's port arrives ready (src/nxt_router.c:5026, src/nxt_status.c:222) -- so it is exactly the question "is the worker still importing". Anything else skips. The worker-discovery loop is bounded at 2s as well, inside the 4s import window rather than the 5s it used to allow, and exhausting it skips too: the fork stalled past the window and there is nothing left to aim at. The docstring now says what the test does. The alert skip was `\b{worker}#{worker} `, which reads the libunit and runtime log prefix pid#tid (src/nxt_unit.c:7319, src/nxt_app_log.c:63) as though the two fields were equal. That holds for a main thread on Linux only; on FreeBSD and macOS nxt_thread_get_tid() is unrelated to the pid (src/nxt_thread_id.h), the skip silently stops matching, and the test fails on the worker's EPIPE teardown noise instead of on its subject. `\b{worker}#\d+ ` keeps the pid scoping, which is the point of scoping it at all, without the assumption. The comment on the set_peer call loses the two paragraphs that restated the commit message and gains the one thing the code does not say: the setter cannot report a failed insert, it logs and leaves ->peer at -1 (src/nxt_port_rpc.c:301), so a pool allocation failure there restores the peerless registration #288 removed -- pre-existing, and shared with every other call site. Verified: build/tests passes (42 subtests, exit 0), and the python test passes three runs in a row; forcing the guard false makes it skip with the observed counter, and breaking the alert pattern makes it fail, so neither check is inert. Co-Authored-By: Claude Opus 5 --- src/nxt_router.c | 19 ++++++------- test/test_python_application.py | 47 +++++++++++++++++++++++++-------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 4179330f5..73ef49876 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -486,18 +486,15 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, * REMOVE_PID the router gets for the prototype carries no stream (it * had already reached READY itself, see nxt_main_process.c) and * nxt_port_rpc_remove_peer() finds nothing to fail, so the attempt's - * pending_processes slot is never given back. With spare 0 that alone - * exceeds max_pending_processes, nxt_router_app_can_start() stays false - * for good, and no later request can even ask for a new prototype -- - * only the start handler does that, and it is what can_start gates. + * pending_processes slot is never given back. * - * With the peer set, nxt_port_rpc_remove_peer() runs - * nxt_router_app_port_error() exactly once (it clears ->peer, deletes - * the stream and frees the registration), which releases the slot and - * fails the ack-waiting requests when no process is left. For a - * prototype start the peer is the main process, which is harmless: its - * death is fatal anyway, and a prototype dying before READY is already - * reported by the stream-bearing REMOVE_PID. + * For a prototype start the peer is the main process, which is + * harmless: its death is fatal anyway, and a prototype dying before + * READY is already reported by the stream-bearing REMOVE_PID. + * + * The setter cannot report a failed insert: it logs and leaves ->peer + * at -1 (src/nxt_port_rpc.c:301), so a pool allocation failure here + * restores the peerless registration -- as at every other call site. */ nxt_port_rpc_ex_set_peer(task, port, app_joint_rpc, dport->pid); diff --git a/test/test_python_application.py b/test/test_python_application.py index f4a066c64..3d49d8772 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -1114,8 +1114,11 @@ def test_python_prototype_killed_mid_start(skip_alert, findall): The app module sleeps in its *import* (test/python/slow_start/wsgi.py), which is what widens the PROCESS_CREATED -> READY window enough to aim - at. A kill that lands outside the window makes the test skip rather - than assert something it did not drive. + at. Landing inside that window is a precondition, not an assertion: + the worker has to be discovered well before the window closes, and + /status has to still count the start as pending right before the kill. + A runner loaded enough to miss either skips rather than reporting a + failure the test did not drive. """ client.load( @@ -1156,22 +1159,41 @@ def _request(): # Wait for the worker to exist: killing the prototype before it has # forked would carry the original start stream and let an unfixed - # router pass this test by the other route. Then a short pause puts - # the worker mid-import: past PROCESS_CREATED, well before READY. + # router pass this test by the other route. Bounded at 2s, inside + # the 4s import window: exhausting it means the fork stalled past + # the window, which leaves nothing for the kill to aim at. Then a + # short pause puts the worker mid-import: past PROCESS_CREATED and + # before READY. workers = [] - for _ in range(100): + for _ in range(40): workers = _child_pids(pid) if workers: break time.sleep(0.05) - assert workers, 'the prototype never forked a worker' + if not workers: + pytest.skip('the prototype forked no worker within the window') time.sleep(0.3) - # The worker is still importing the module. Its own teardown noise - # is scoped to its pid below rather than skipped by shape, so an - # alert from any other process still fails the test. + # The worker must still be importing for the kill to land where this + # test aims it. "starting" is app->pending_processes, which the + # router gives back only once the worker's port arrives ready, so + # anything but the one pending start means the window has closed -- + # the request would then be served with 200 and the assertions below + # would report the wrong thing. + starting = client.conf_get('/status')['applications']['slow_start'][ + 'processes' + ]['starting'] + + if starting != 1: + pytest.skip( + f'the start left the import window: starting == {starting}' + ) + + # Its own teardown noise is scoped to the worker's pid below rather + # than skipped by shape, so an alert from any other process still + # fails the test. subprocess.call(['kill', '-9', pid]) skip_alert(fr'process {pid} exited on signal 9') @@ -1183,8 +1205,11 @@ def _request(): # prototype -- which the kill already closed. All of that is # the pre-existing consequence of killing a prototype, not # something this test's subject produces; it happens on the - # unfixed router too. - skip_alert(fr'\b{worker}#{worker} ') + # unfixed router too. Both log prefixes are pid#tid + # (src/nxt_unit.c:7319, src/nxt_app_log.c:63), and the two are + # equal only for a main thread on Linux -- scope by the pid, + # which is the point of this skip, and let the tid be. + skip_alert(fr'\b{worker}#\d+ ') # The kill retires the start RPC, which releases the slot and -- no # process being left -- fails the waiting request instead of letting From 44a10ff38428ccc2563d009131e39ffeb1feb920 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 13:08:56 +0200 Subject: [PATCH 094/391] isolation: reject "rootfs" + credential ns without a mount ns An app with "isolation.rootfs" and "namespaces": {"credential": true} but no "namespaces": {"mount": true} was accepted by the validator and then always failed at prototype start, with only "Failed to apply new configuration." on the control API. The child unshares CLONE_NEWUSER without CLONE_NEWNS (src/nxt_isolation.c:438-466, unshare(2) at src/nxt_process.c:526), so it holds capabilities in a new user namespace while still living in the parent's mount namespace, which is owned by the initial user namespace. mount(2) needs CAP_SYS_ADMIN in the namespace owning the mount namespace, so every automount in nxt_isolation_prepare_rootfs() (src/nxt_isolation.c:926) gets EPERM at src/nxt_fs_mount.c:93 and the prototype dies. That is a kernel invariant, not something Unit can fix at runtime, so refuse it at configuration time instead. The refusal is narrower than "rootfs + credential": the rootfs switch itself needs no mount namespace -- nxt_isolation_change_root() (src/nxt_isolation.c:1058) falls back to chroot(2), which needs only CAP_SYS_CHROOT in the new user namespace -- so a config that disables every applicable automount does start and serve, and stays valid. The new check therefore fires only when a mount would be attempted: "automount.procfs" or "automount.tmpfs" enabled (unconditional builtins, src/nxt_isolation.c:700 and :740), or "automount.language_deps" enabled when the module declares language dependency mounts (lang mounts all carry deps = 1, src/nxt_main_process.c:1768, and are skipped when "language_deps" is false, src/nxt_isolation.c:926). That last arm asks what the module declares, not what the runtime would end up mounting: the same loop also skips a bind mount whose source is missing on the host (src/nxt_isolation.c:930), so a module whose every declared dependency happens to be absent would in fact have started. Answering that here would mean stat()ing host paths from the validator, which runs in the controller as the unitd user while the prototype mounts as root, so it would be wrong in the other direction just as easily. It runs from nxt_conf_vldt_app(), after the language module is resolved, because lang->mounts is what the "language_deps" arm asks about; the controller inherits the populated rt->languages, which is filled before the controller is forked (src/nxt_main_process.c:1782-1790). The error names every automount that would be attempted, and spells out the "automount" object that turns them off, so that a config with more than one of them enabled is fixed in a single edit rather than in one round trip per automount. Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Fable 5.1 --- src/nxt_conf_validation.c | 191 +++++++++++++++++++++++++++++++++- test/test_php_isolation.py | 128 +++++++++++++++++++++++ test/test_python_isolation.py | 50 +++++++++ 3 files changed, 368 insertions(+), 1 deletion(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 7bf263632..8d50c3deb 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -257,6 +257,12 @@ static nxt_int_t nxt_conf_vldt_rootfs_path(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); #endif +#if (NXT_HAVE_ISOLATION_ROOTFS) && (NXT_HAVE_CLONE_NEWUSER) \ + && (NXT_HAVE_CLONE_NEWNS) +static nxt_int_t nxt_conf_vldt_isolation_mounts(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, nxt_app_lang_module_t *lang); +#endif + #if (NXT_HAVE_NJS) static nxt_int_t nxt_conf_vldt_js_module(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); @@ -3469,7 +3475,17 @@ nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, return ret; } - return types[lang->type].validator(vldt, value, types[lang->type].members); + ret = types[lang->type].validator(vldt, value, types[lang->type].members); + if (ret != NXT_OK) { + return ret; + } + +#if (NXT_HAVE_ISOLATION_ROOTFS) && (NXT_HAVE_CLONE_NEWUSER) \ + && (NXT_HAVE_CLONE_NEWNS) + return nxt_conf_vldt_isolation_mounts(vldt, value, lang); +#else + return NXT_OK; +#endif } @@ -4051,6 +4067,179 @@ nxt_conf_vldt_isolation(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +#if (NXT_HAVE_ISOLATION_ROOTFS) && (NXT_HAVE_CLONE_NEWUSER) \ + && (NXT_HAVE_CLONE_NEWNS) + +static nxt_bool_t +nxt_conf_vldt_boolean_member(nxt_conf_value_t *object, const nxt_str_t *name, + nxt_bool_t dflt) +{ + nxt_conf_value_t *member; + + if (object == NULL) { + return dflt; + } + + member = nxt_conf_get_object_member(object, name, NULL); + + if (member == NULL || nxt_conf_type(member) != NXT_CONF_BOOLEAN) { + return dflt; + } + + return nxt_conf_get_boolean(member); +} + + +/* + * A prototype that enters a new user namespace (CLONE_NEWUSER, from + * "namespaces": {"credential": true}) without a new mount namespace + * (CLONE_NEWNS, from "namespaces": {"mount": true}) stays in the parent's + * mount namespace, which is owned by the initial user namespace. mount(2) + * there requires CAP_SYS_ADMIN in *that* namespace, which the child does not + * have, so the automount loop in nxt_isolation_prepare_rootfs() + * (src/nxt_isolation.c:926) fails with EPERM at src/nxt_fs_mount.c:93 and the + * prototype dies before it can serve -- with only "Failed to apply new + * configuration." on the control API. This is a kernel invariant, so refuse + * the combination here instead. + * + * The rootfs switch itself does not need CLONE_NEWNS: + * nxt_isolation_change_root() (src/nxt_isolation.c:1058) falls back to + * chroot(2), which needs only CAP_SYS_CHROOT in the new user namespace. So + * refuse only the configurations that have a mount to perform: the "tmpfs" + * and "procfs" automounts (unconditional builtins, added in + * nxt_isolation_set_lang_mounts(), src/nxt_isolation.c:703 and :729), and the + * "language_deps" automount when the module declares language dependency + * mounts (every lang mount carries deps = 1, src/nxt_main_process.c:1768, and + * is skipped when "language_deps" is false, src/nxt_isolation.c:926). A + * config that disables every applicable automount does start and serve, and + * stays valid. + * + * The "language_deps" arm asks what the module declares, not what the runtime + * would end up mounting: the same loop also skips a bind mount whose source is + * missing on the host (src/nxt_isolation.c:930), so a module whose every + * declared dependency happens to be absent -- copied into the rootfs already, + * say -- would in fact have started. Answering that here would mean stat()ing + * host paths from the validator, which runs in the controller as the unitd + * user while the prototype mounts as root, so it would be wrong in the other + * direction just as easily. Declared-but-absent is therefore the line: it + * needs no filesystem access, it does not change under the app's feet, and the + * error names the automount to switch off. + */ +static nxt_int_t +nxt_conf_vldt_isolation_mounts(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, nxt_app_lang_module_t *lang) +{ + u_char *p, *q; + u_char names_buf[64], disable_buf[96]; + nxt_str_t names, disable; + nxt_uint_t i, n; + nxt_conf_value_t *isolation, *namespaces, *automount; + const nxt_str_t *enabled[3]; + + static const nxt_str_t isolation_str = nxt_string("isolation"); + static const nxt_str_t namespaces_str = nxt_string("namespaces"); + static const nxt_str_t automount_str = nxt_string("automount"); + static const nxt_str_t rootfs_str = nxt_string("rootfs"); + static const nxt_str_t credential_str = nxt_string("credential"); + static const nxt_str_t mount_str = nxt_string("mount"); + static const nxt_str_t tmpfs_str = nxt_string("tmpfs"); + static const nxt_str_t procfs_str = nxt_string("procfs"); + static const nxt_str_t langdeps_str = nxt_string("language_deps"); + + isolation = nxt_conf_get_object_member(value, &isolation_str, + NULL); + if (isolation == NULL + || nxt_conf_type(isolation) != NXT_CONF_OBJECT + || nxt_conf_get_object_member(isolation, &rootfs_str, + NULL) == NULL) + { + return NXT_OK; + } + + namespaces = nxt_conf_get_object_member(isolation, + &namespaces_str, + NULL); + if (namespaces == NULL || nxt_conf_type(namespaces) != NXT_CONF_OBJECT) { + return NXT_OK; + } + + if (!nxt_conf_vldt_boolean_member(namespaces, &credential_str, 0) + || nxt_conf_vldt_boolean_member(namespaces, &mount_str, 0)) + { + return NXT_OK; + } + + automount = nxt_conf_get_object_member(isolation, + &automount_str, NULL); + if (automount != NULL && nxt_conf_type(automount) != NXT_CONF_OBJECT) { + automount = NULL; + } + + /* + * Collect every automount that would be attempted, so that one error + * names them all and the operator can turn them off in a single edit. + */ + + n = 0; + + if (nxt_conf_vldt_boolean_member(automount, &procfs_str, 1)) { + enabled[n++] = &procfs_str; + } + + if (nxt_conf_vldt_boolean_member(automount, &tmpfs_str, 1)) { + enabled[n++] = &tmpfs_str; + } + + if (nxt_conf_vldt_boolean_member(automount, &langdeps_str, 1) + && lang->mounts != NULL && lang->mounts->nelts > 0) + { + enabled[n++] = &langdeps_str; + } + + if (n == 0) { + return NXT_OK; + } + + p = names_buf; + q = disable_buf; + + for (i = 0; i < n; i++) { + p = nxt_sprintf(p, names_buf + sizeof(names_buf), "%s\"%V\"", + (i == 0) ? (u_char *) "" : + (i + 1 == n) ? (u_char *) " and " : (u_char *) ", ", + enabled[i]); + + q = nxt_sprintf(q, disable_buf + sizeof(disable_buf), + "%s\"%V\": false", + (i == 0) ? (u_char *) "" : (u_char *) ", ", + enabled[i]); + } + + names.start = names_buf; + names.length = p - names_buf; + + disable.start = disable_buf; + disable.length = q - disable_buf; + + return nxt_conf_vldt_error(vldt, + "The \"isolation\" object sets \"rootfs\" with " + "\"namespaces\": {\"credential\": true} but " + "without \"namespaces\": {\"mount\": true}; a " + "process in a new user namespace but in the " + "parent mount namespace cannot mount the %V " + "%s (EPERM), so the application could never " + "start. Set \"namespaces\": {\"mount\": true}, " + "drop \"namespaces\": {\"credential\": true}, " + "or set \"automount\": {%V}.", + &names, + (n == 1) ? (u_char *) "automount" + : (u_char *) "automounts", + &disable); +} + +#endif + + #if (NXT_HAVE_CLONE_NEWUSER) static nxt_int_t diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index f248da417..918975c00 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -83,3 +83,131 @@ def test_php_isolation_rootfs_extensions(is_su, require, temp_dir): assert 'json' in extensions, 'json in extensions list' assert 'unit' in extensions, 'unit in extensions list' + + +def test_php_isolation_rootfs_credential_without_mount(is_su, require, + temp_dir): + if not is_su: + require( + {'features': {'isolation': ['unprivileged_userns_clone', 'user']}} + ) + else: + require({'features': {'isolation': ['user']}}) + + client.load('phpinfo') + + # A new user namespace without a new mount namespace cannot mount + # anything: the child stays in the parent's mount namespace, owned by + # the initial user namespace. The automounts default to on, so this + # config could never start; it must be refused at configuration time. + resp = client.conf( + {'rootfs': temp_dir, 'namespaces': {'credential': True}}, + 'applications/phpinfo/isolation', + ) + + assert 'error' in resp, 'credential without mount rejected' + + detail = resp.get('detail', '') + + assert 'rootfs' in detail, 'detail names rootfs' + assert 'credential' in detail, 'detail names credential' + assert 'mount' in detail, 'detail names mount' + assert 'automount' in detail, 'detail names automount' + + # The remedy must name the knobs to turn off, not just "the automount + # options". "procfs" and "tmpfs" are unconditional builtins, so both + # arms fire here whatever the module declares. + assert '"procfs": false' in detail, 'detail names the procfs knob' + assert '"tmpfs": false' in detail, 'detail names the tmpfs knob' + + # An explicit "mount": false is the same thing. + resp = client.conf( + { + 'rootfs': temp_dir, + 'namespaces': {'credential': True, 'mount': False}, + }, + 'applications/phpinfo/isolation', + ) + + assert 'error' in resp, 'explicit mount false rejected' + + # Without "credential" the config is fine -- the prototype keeps the + # main process' privileges and mounts in the parent mount namespace. + assert 'success' in client.conf( + {'rootfs': temp_dir, 'namespaces': {'credential': False}}, + 'applications/phpinfo/isolation', + ), 'no credential accepted' + + +def test_php_isolation_rootfs_credential_with_mount(is_su, require, temp_dir): + require( + { + 'features': { + 'isolation': ( + ['user', 'mnt', 'pid'] + if is_su + else ['unprivileged_userns_clone', 'user', 'mnt', 'pid'] + ) + } + } + ) + + client.load( + 'phpinfo', + isolation={ + 'rootfs': temp_dir, + 'namespaces': {'credential': True, 'mount': True, 'pid': True}, + }, + ) + + assert 'success' in client.conf( + '"/app/php/phpinfo"', 'applications/phpinfo/root' + ) + assert 'success' in client.conf( + '"/app/php/phpinfo"', 'applications/phpinfo/working_directory' + ) + + assert client.get()['status'] == 200, 'credential + mount serves' + + +def test_php_isolation_rootfs_credential_no_automount(is_su, require, + temp_dir): + require( + { + 'features': { + 'isolation': ( + ['user'] + if is_su + else ['unprivileged_userns_clone', 'user'] + ) + } + } + ) + + # With every automount disabled nothing is mounted: the rootfs switch + # falls back to chroot(2), which needs only CAP_SYS_CHROOT in the new + # user namespace. This works, so the validator must keep accepting it. + # + # "language_deps" is off here too, so the case stays valid whether or + # not the PHP module declares dependency mounts of its own. + client.load( + 'phpinfo', + isolation={ + 'rootfs': temp_dir, + 'namespaces': {'credential': True}, + 'automount': { + 'procfs': False, + 'tmpfs': False, + 'language_deps': False, + }, + }, + ) + + assert 'success' in client.conf( + '"/app/php/phpinfo"', 'applications/phpinfo/root' + ) + assert 'success' in client.conf( + '"/app/php/phpinfo"', 'applications/phpinfo/working_directory' + ) + + assert client.get()['status'] == 200, 'no automount serves' diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index 26679e6c4..87164cb36 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -127,6 +127,56 @@ def test_python_isolation_rootfs_no_language_deps(require, temp_dir): ), 'language_deps unmount' +def test_python_isolation_rootfs_credential_language_deps( + is_su, require, temp_dir +): + if not is_su: + require( + {'features': {'isolation': ['unprivileged_userns_clone', 'user']}} + ) + else: + require({'features': {'isolation': ['user']}}) + + client.load('empty') + + # A new user namespace without a new mount namespace cannot mount + # anything. With "procfs" and "tmpfs" off the only mounts left are the + # language dependencies, which this module always declares (auto/modules/ + # python emits at least the stdlib directory), so the config still could + # never start and must be refused -- naming "language_deps", and only + # "language_deps", as the automount to switch off. + resp = client.conf( + { + 'rootfs': temp_dir, + 'namespaces': {'credential': True}, + 'automount': {'procfs': False, 'tmpfs': False}, + }, + 'applications/empty/isolation', + ) + + assert 'error' in resp, 'language_deps automount rejected' + + detail = resp.get('detail', '') + + assert '"language_deps": false' in detail, 'detail names the knob' + assert 'procfs' not in detail, 'disabled procfs not named' + assert 'tmpfs' not in detail, 'disabled tmpfs not named' + + # Nothing left to mount, so the very same config becomes valid. + assert 'success' in client.conf( + { + 'rootfs': temp_dir, + 'namespaces': {'credential': True}, + 'automount': { + 'procfs': False, + 'tmpfs': False, + 'language_deps': False, + }, + }, + 'applications/empty/isolation', + ), 'every automount off accepted' + + def test_python_isolation_procfs(require, temp_dir): require({'privileged_user': True}) From 275e19a0ebab8c5444a7c9cc6a164bcd79c5f11a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 18:09:53 +0200 Subject: [PATCH 095/391] capability: keep running when capget() is filtered A non-root unitd asks the kernel for its own capabilities so it can decide whether "user", "group" and "rootfs" can be honoured. When a syscall filter denies capget() -- a systemd "SystemCallFilter=" allowlist, a hand-rolled seccomp profile -- that question has no answer, but it never had to be answered for Unit to start: without any capability at all it runs perfectly well, just without user switching. Refusing to start turned an operator's hardening choice into an outage. Treat the two errnos such a filter conventionally returns, EPERM and ENOSYS, as "capabilities unknown": warn, leave both flags clear, and continue. Every other errno means the call itself is malformed rather than blocked -- EINVAL after the version probe already succeeded, EFAULT -- so those keep aborting, because a privilege downgrade caused by a bug in Unit is worse than a hard failure. The warning names the consequences rather than the syscall, since the consequences are what an operator has to act on: user and group switching and "rootfs" isolation are disabled for applications that do not enable the "credential" namespace -- a NEWUSER namespace re-enables both -- and applications run as unitd's own uid. That is a downgrade, not an escalation, but it would otherwise be a silent one: with setid clear the runtime never resolves credentials and never calls setgid()/setuid(), while an application's "user" is only string-compared against --user, so a misconfigured deployment would look correct in the log. nxt_capability_set() runs from nxt_runtime_conf_init(), before nxt_runtime_start() opens the log file, so on its own the warning would reach stderr only and a daemonised unitd -- the normal deployment -- would record nothing at all. A flag on the runtime carries it to the first point where unit.log exists. The test reproduces the deployment rather than the branch: an LD_PRELOAD shim interposes syscall() and fails SYS_capget with a chosen errno, so the whole startup path runs against a unitd that genuinely cannot read its own capabilities. It records every caller's pid, which is what shows the shim reaches only the process under test and is inherited harmlessly by the application. The record path is strdup()ed because nxt_process_title() overwrites the environ block. The test needs a unitd that actually calls capget(), which a root one never does, so every existing CI leg skips it -- the file sits inside "test", so it looked covered, and had never once run. The python leg repeats it without sudo, after the full suite so a failure here cannot hide the language results. pytest-3 is named absolutely because setup-python puts a toolcache bin directory ahead of /usr/bin for an unprivileged step and not for the sudo'd one. docs/changes.xml gains the companion 1.36.2 module stanza along with the entry: dpkg-source fails for each module whose changelog has no entry for the version being built, so the packages cannot be built without it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VXRyL2dx3yYBqaaYsdpuQE --- .github/workflows/build-test.yml | 22 ++ CHANGES | 13 ++ docs/changes.xml | 45 ++++ src/nxt_capability.c | 41 +++- src/nxt_capability.h | 1 + src/nxt_runtime.c | 13 ++ test/capget_filter.c | 172 ++++++++++++++++ test/test_capget_fallback.py | 344 +++++++++++++++++++++++++++++++ 8 files changed, 649 insertions(+), 2 deletions(-) create mode 100644 test/capget_filter.c create mode 100644 test/test_capget_fallback.py diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 2305b98d5..52ab2b969 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -543,6 +543,28 @@ jobs: fi if: steps.metadata.outputs.module != 'wasm' + # test_capget_fallback.py needs a unitd that actually calls capget(), + # which a root one never does, so every leg above skips it -- the file + # is inside "test" but has never once run. Repeat it here without + # sudo. The python leg is the one chosen because the test drives a + # real application to show it inherits the shim harmlessly, and this + # leg is where a language module exists. + # + # pytest-3 is named absolutely: setup-python puts a toolcache bin + # directory ahead of /usr/bin for this step and not for the sudo'd one + # above, and both must run the pytest that "apt install python3-pytest" + # provided. + # + # !cancelled() rather than the implicit success(): this is the only leg + # that ever runs the file, and gating it on the whole sudo'd suite above + # would let one unrelated flake silently take that coverage away again -- + # the very problem this step exists to fix. + - name: Run capget fallback tests unprivileged + run: | + export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" + /usr/bin/pytest-3 --print-log test/test_capget_fallback.py + if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} + # C unit-test suite (src/test/ -> build/tests): HTTP parser, port/IPC # fault-injection, lvlhsh, mp, rbtree, base64, utf8, clone creds, etc. # Previously compiled but never executed in CI; runs here on every PR as its diff --git a/CHANGES b/CHANGES index 7f02dc982..7d6f86316 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,17 @@ +Changes with FreeUnit 1.36.2 30 Aug 2026 + + *) Bugfix: do not exit when the capget() syscall is denied. A syscall + filter that omits capget -- a systemd "SystemCallFilter=" allowlist, + a hand-written seccomp profile -- made a non-root unitd fail to start + with "failed to get process capabilities". Unit now warns and runs + without assuming any capability: user and group switching and + "rootfs" isolation are disabled for applications that do not enable + the "credential" namespace, and applications run as unitd's own uid. + Capabilities the process may still hold are not dropped, since Unit + does not call capset(). Every other capget() failure remains fatal. + + Changes with FreeUnit 1.36.1 28 Aug 2026 *) Feature: configuration validation errors now say where they are. A diff --git a/docs/changes.xml b/docs/changes.xml index ab0b746c9..b091589ec 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -5,6 +5,51 @@ + + + + +do not exit when the capget() syscall is denied. A syscall filter that +omits capget -- a systemd "SystemCallFilter=" allowlist, a hand-written +seccomp profile -- made a non-root unitd fail to start with "failed to get +process capabilities". Unit now warns and runs without assuming any +capability: user and group switching and "rootfs" isolation are disabled +for applications that do not enable the "credential" namespace, and +applications run as unitd's own uid. Capabilities the process may +still hold are not dropped, since Unit does not call capset(). Every +other capget() failure remains fatal. + + + + + + + + + + +FreeUnit updated to 1.36.2. + + + + + + setid == 0); - if (geteuid() == 0) { + if (nxt_euid == 0) { cap->setid = 1; cap->chroot = 1; return NXT_OK; @@ -77,6 +77,7 @@ nxt_capability_linux_get_version(void) static nxt_int_t nxt_capability_specific_set(nxt_task_t *task, nxt_capabilities_t *cap) { + nxt_err_t err; struct __user_cap_data_struct *val, data[2]; struct __user_cap_header_struct hdr; @@ -100,7 +101,43 @@ nxt_capability_specific_set(nxt_task_t *task, nxt_capabilities_t *cap) hdr.pid = nxt_pid; if (nxt_slow_path(nxt_capget(&hdr, val) == -1)) { - nxt_alert(task, "failed to get process capabilities: %E", nxt_errno); + err = nxt_errno; + + /* + * A syscall filter that does not allow capget() reports it + * through SECCOMP_RET_ERRNO, conventionally as EPERM or ENOSYS. + * Capabilities then cannot be observed at all, which is not a + * reason to refuse to start: leave both flags clear and run the + * way an ordinary unprivileged unitd already does. + * + * Any other errno means the call itself is wrong -- EINVAL, + * which a working version probe would have prevented, or EFAULT + * for a bad pointer -- so it is a bug in Unit rather than a + * policy decision made by the operator. Keep failing closed + * there, rather than silently downgrading privileges. + */ + + if (err == NXT_EPERM || err == NXT_ENOSYS) { + nxt_log(task, NXT_LOG_WARN, "capget() failed %E; process " + "capabilities are unknown and will not be used: user and " + "group switching and \"rootfs\" isolation are disabled " + "for applications that do not enable the \"credential\" " + "namespace, and applications run as uid %d", + err, (int) nxt_euid); + + /* + * The log file is not open yet -- this runs from + * nxt_runtime_conf_init() -- so the warning above reaches + * stderr only. Record it, and nxt_runtime_start() repeats it + * once unit.log exists, or a daemonised unitd would keep no + * trace of running without knowing its own capabilities. + */ + cap->unknown = 1; + + return NXT_OK; + } + + nxt_alert(task, "failed to get process capabilities: %E", err); return NXT_ERROR; } diff --git a/src/nxt_capability.h b/src/nxt_capability.h index 1575d4094..030e6ccab 100644 --- a/src/nxt_capability.h +++ b/src/nxt_capability.h @@ -9,6 +9,7 @@ typedef struct { uint8_t setid; /* 1 bit */ uint8_t chroot; /* 1 bit */ + uint8_t unknown; /* 1 bit */ } nxt_capabilities_t; diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 4e1aa2f63..59013efbf 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -357,6 +357,19 @@ nxt_runtime_start(nxt_task_t *task, void *obj, void *data) goto fail; } + if (rt->capabilities.unknown) { + /* + * nxt_capability_set() could only reach stderr: it runs from + * nxt_runtime_conf_init(), before the log file exists. Repeat it + * here so the record survives in unit.log too. + */ + nxt_log(task, NXT_LOG_WARN, "capget() failed; process " + "capabilities are unknown and will not be used: user and " + "group switching and \"rootfs\" isolation are disabled " + "for applications that do not enable the \"credential\" " + "namespace, and applications run as uid %d", (int) nxt_euid); + } + if (nxt_runtime_event_engine_change(task, rt) != NXT_OK) { goto fail; } diff --git a/test/capget_filter.c b/test/capget_filter.c new file mode 100644 index 000000000..a9ef04ab8 --- /dev/null +++ b/test/capget_filter.c @@ -0,0 +1,172 @@ +/* + * LD_PRELOAD shim that makes capget(2) fail the way a syscall filter + * makes it fail, for test_capget_fallback.py. + * + * src/nxt_capability.c reaches the kernel through the nxt_capget() + * macro, which expands to glibc's syscall() wrapper -- an ordinary + * interposable symbol, not a compiler builtin or an inline "syscall" + * instruction. So a preloaded definition of syscall() shadows it and + * we can inject an errno without needing seccomp, a privileged + * helper, or a container. + * + * The interposition is total: every syscall() in the process, from + * every library, comes through here. Only SYS_capget is ever + * answered locally, and only when NXT_TEST_CAPGET_ERRNO names an + * errno; everything else is forwarded to the real symbol untouched. + * + * Set NXT_TEST_CAPGET_RECORD to a file path to have every pid whose + * capget() we failed appended to it, one per line. The test uses + * that to show which processes the filter actually reached -- the + * environment is inherited by unitd's children, so "no other process + * was affected" has to be demonstrated rather than assumed. + * + * That path is copied, not kept as the getenv() pointer, and the copy + * is what makes the demonstration mean anything. getenv() returns a + * pointer into the contiguous argv+environ block, and + * nxt_process_title() (src/nxt_process_title.c) writes the process + * title over that block and zero-pads the remainder, so in every + * process unitd forks the inherited pointer would read as "" and + * open("") would fail silently. The main process looked correct only + * by accident of ordering: nxt_capability_set() runs from + * nxt_runtime_conf_init(), before nxt_main_process_title() + * (src/nxt_main_process.c:96) rewrites the area. A record file that + * no child could have written to proves nothing about what the + * children did. + * + * Build: + * cc -shared -fPIC -o capget_filter.so capget_filter.c -ldl + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * glibc declares syscall() variadic, but it is an assembly stub that + * only moves registers: it never walks a va_list, so calling it + * through a fixed six-argument prototype is what every syscall + * interposer does and is correct on every Linux ABI Unit builds for. + */ +typedef long (*nxt_syscall_fn)(long, long, long, long, long, long, long); + +static nxt_syscall_fn nxt_real_syscall; +static int nxt_capget_errno; /* 0: filter disabled */ +static char *nxt_record_path; /* strdup()ed, see above */ + + +static void +nxt_shim_init(void) +{ + const char *value; + + nxt_real_syscall = (nxt_syscall_fn) dlsym(RTLD_NEXT, "syscall"); + + value = getenv("NXT_TEST_CAPGET_ERRNO"); + + if (value != NULL && *value != '\0') { + nxt_capget_errno = (int) strtol(value, NULL, 10); + } + + value = getenv("NXT_TEST_CAPGET_RECORD"); + + if (nxt_record_path == NULL && value != NULL && *value != '\0') { + nxt_record_path = strdup(value); + } +} + + +__attribute__((constructor)) +static void +nxt_shim_ctor(void) +{ + nxt_shim_init(); +} + + +static void +nxt_shim_record(void) +{ + int fd, len; + char buf[32]; + + if (nxt_record_path == NULL) { + return; + } + + /* + * open()/write()/close() are their own glibc stubs, not calls to + * syscall(), so this cannot recurse back into the shim. + */ + + fd = open(nxt_record_path, O_WRONLY | O_CREAT | O_APPEND, 0644); + + if (fd < 0) { + return; + } + + len = snprintf(buf, sizeof(buf), "%ld\n", (long) getpid()); + + if (len > 0) { + (void) !write(fd, buf, (size_t) len); + } + + close(fd); +} + + +long +syscall(long number, ...) +{ + long a0, a1, a2, a3, a4, a5; + va_list ap; + + if (nxt_real_syscall == NULL) { + /* + * A constructor in another preloaded object may run before + * ours and call syscall() from it. + */ + nxt_shim_init(); + + if (nxt_real_syscall == NULL) { + /* + * Nothing to forward to. Fail loudly: silently returning + * an error here would corrupt an unrelated syscall and + * make the test lie about what it exercised. + */ + _exit(127); + } + } + + /* + * Reading six arguments when the caller passed fewer yields + * garbage in the unused ones, which the kernel ignores for every + * syscall that does not declare them. + */ + + va_start(ap, number); + a0 = va_arg(ap, long); + a1 = va_arg(ap, long); + a2 = va_arg(ap, long); + a3 = va_arg(ap, long); + a4 = va_arg(ap, long); + a5 = va_arg(ap, long); + va_end(ap); + +#ifdef SYS_capget + if (number == SYS_capget && nxt_capget_errno != 0) { + nxt_shim_record(); + errno = nxt_capget_errno; + return -1; + } +#endif + + return nxt_real_syscall(number, a0, a1, a2, a3, a4, a5); +} diff --git a/test/test_capget_fallback.py b/test/test_capget_fallback.py new file mode 100644 index 000000000..841d6a9e1 --- /dev/null +++ b/test/test_capget_fallback.py @@ -0,0 +1,344 @@ +"""Unit keeps running when a syscall filter denies capget(). + +A hardened deployment -- a systemd "SystemCallFilter=" allowlist, a +hand-written seccomp profile -- can make capget(2) fail for a non-root +unitd. Since 'capability: keep running when capget() is filtered' that +is a warning rather than an outage for the two errnos such a filter +conventionally returns, EPERM and ENOSYS; every other errno still +aborts, because a privilege downgrade caused by a bug in Unit is worse +than a hard failure. + +Reproducing a seccomp filter in a test would need privileges the suite +does not have, so the errno is injected instead: src/nxt_capability.c +calls the kernel through the nxt_capget() macro, which expands to +glibc's syscall() wrapper, and that is an ordinary interposable symbol. +test/capget_filter.c preloads a definition of it -- see that file for +what it does and does not touch. + +Inheritance is accepted rather than papered over. Unit's language +modules are fork()ed, never exec()ed, so the application process +inherits the shim already mapped and already armed -- the errno was +read by the shim's constructor in the main process, long before the +fork. Clearing LD_PRELOAD from the application's "environment" would +therefore disarm nothing; it would only hide the fact. + +That inheritance costs nothing, and the test proves it rather than +asserting it: the shim appends the pid of every process whose capget() +it failed to a record file, and the test requires that set to be +exactly the unitd main process. It can be, because +nxt_capability_set() has a single caller, nxt_runtime_conf_init(), +which runs in the main process before anything is forked. For that +record to be evidence and not an empty file, the shim keeps a copy of +the record path rather than the getenv() pointer -- unitd overwrites +the argv+environ block with its process title, so the pointer alone +would leave every child unable to record anything at all. See +capget_filter.c. + +/proc//environ deliberately plays no part in that proof. It +cannot: nxt_process_arguments() takes the contiguous argv+environ +string area for the process title, so every Unit process has an +environ region full of title text and NUL padding while the real +environment lives on the heap. Measured here, the main process shows +the tail of its own argv and the application shows 4807 NUL bytes -- +LD_PRELOAD is invisible in both, in a run where the shim demonstrably +ran in the main process. What the application's /proc//maps +shows instead is the shim object itself, mapped, which is the thing +that would actually matter. +""" + +import errno +import json +import os +import re +import shutil +import signal +import subprocess +import tempfile +import time +from pathlib import Path + +import pytest + +from unit.http import HTTP1 +from unit.option import option +from unit.utils import public_dir, waitforfiles + +client = HTTP1() + +WARNING = r'\[warn\].+capget\(\) failed.+capabilities are unknown' +ALERT = r'\[alert\].+failed to get process capabilities' + +# The two errnos a syscall filter conventionally returns for a denied +# call. nxt_capability.c accepts both, so both belong in the test. +FILTERED = [errno.EPERM, errno.ENOSYS] + + +@pytest.fixture(scope='module', autouse=True) +def requirements(): + if option.system != 'Linux': + pytest.skip('capget() is Linux only') + + if option.is_privileged: + pytest.skip('a root unitd never calls capget()') + + +@pytest.fixture(scope='module') +def shim(requirements): + """Build test/capget_filter.c into a preloadable object. + + The suite has no convention for compiling C helpers -- nothing under + test/ builds any -- so rather than invent a build system this + compiles at test time with cc(1) and skips when there is none. + + Depends on requirements() rather than trusting fixture ordering: + both are module-scoped, and the link needs -ldl, so on a platform + that should skip the build would otherwise fail first and turn the + skip into an error. + """ + compiler = shutil.which('cc') or shutil.which('gcc') + + if compiler is None: + pytest.skip('requires a C compiler') + + source = f'{option.test_dir}/capget_filter.c' + outdir = tempfile.mkdtemp(prefix='unit-test-capget-') + library = f'{outdir}/capget_filter.so' + + build = subprocess.run( + [compiler, '-shared', '-fPIC', '-O1', '-o', library, source, '-ldl'], + check=False, + capture_output=True, + ) + + assert build.returncode == 0, ( + f'failed to build the capget shim:\n{build.stderr.decode()}' + ) + + yield library + + shutil.rmtree(outdir, ignore_errors=True) + + +class Unitd: + """A unitd of our own, since conftest's shared one has no way to + take an environment.""" + + def __init__(self, shim, errno): + builddir = f'{option.current_dir}/build' + self.unitd = f'{builddir}/sbin/unitd' + + if not Path(self.unitd).is_file(): + pytest.skip('could not find unitd') + + self.dir = tempfile.mkdtemp(prefix='unit-test-capget-') + public_dir(self.dir) + + Path(f'{self.dir}/state').mkdir() + + self.log = f'{self.dir}/unit.log' + self.stderr = f'{self.dir}/stderr.log' + self.record = f'{self.dir}/capget.pids' + self.control = f'{self.dir}/control.unit.sock' + self.listener = f'{self.dir}/app.sock' + self.pidfile = f'{self.dir}/unit.pid' + + env = os.environ.copy() + + # Same UNIT_PYTHONHOME dance as conftest.unit_run(): PYTHONHOME + # must not be set for the pytest interpreter itself. + pythonhome = env.pop('UNIT_PYTHONHOME', None) + if pythonhome: + env['PYTHONHOME'] = pythonhome + + env['LD_PRELOAD'] = shim + env['NXT_TEST_CAPGET_ERRNO'] = str(errno) + env['NXT_TEST_CAPGET_RECORD'] = self.record + + # unitd's stderr is kept apart from unit.log on purpose. The + # first warning is written from nxt_runtime_conf_init(), before + # nxt_runtime_log_files_create() dup2()s the log file over fd 2, + # so it can only land here; unit.log then holds exactly the copy + # nxt_runtime_start() repeats once the file exists. Merging the + # two -- which is what conftest does -- would make it impossible + # to tell which of the two emissions a match came from. + self.stderr_file = open(self.stderr, 'w', encoding='utf-8') + + self.process = subprocess.Popen( + [ + self.unitd, + '--no-daemon', + '--modulesdir', + f'{builddir}/lib/unit/modules', + '--statedir', + f'{self.dir}/state', + '--pid', + self.pidfile, + '--log', + self.log, + '--control', + f'unix:{self.control}', + '--tmpdir', + self.dir, + ], + stdout=subprocess.DEVNULL, + stderr=self.stderr_file, + start_new_session=True, + env=env, + ) + + def stop(self): + if self.process.poll() is None: + try: + os.killpg(self.process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + self.process.wait(timeout=30) + self.stderr_file.close() + shutil.rmtree(self.dir, ignore_errors=True) + + def read(self, path): + return Path(path).read_text(encoding='utf-8', errors='ignore') + + def filtered_pids(self): + if not Path(self.record).exists(): + return set() + + return {int(p) for p in self.read(self.record).split()} + + +@pytest.fixture() +def unitd_factory(shim): + started = [] + + def factory(errno): + instance = Unitd(shim, errno) + started.append(instance) + return instance + + yield factory + + for instance in started: + instance.stop() + + +def app_conf(appdir): + """A one-line application that answers with its own pid, so the + /proc lookups below cannot land on the wrong process.""" + modules = option.available['modules'] + + if 'python' in modules: + Path(f'{appdir}/wsgi_pid.py').write_text( + 'import os\n' + '\n' + '\n' + 'def application(environ, start_response):\n' + ' body = str(os.getpid()).encode()\n' + " start_response('200 OK'," + " [('Content-Length', str(len(body)))])\n" + ' return [body]\n', + encoding='utf-8', + ) + + return { + "type": "python", + "processes": {"spare": 0}, + "path": appdir, + "module": "wsgi_pid", + } + + if 'php' in modules: + Path(f'{appdir}/index.php').write_text( + ' Date: Tue, 1 Sep 2026 00:05:24 +0200 Subject: [PATCH 096/391] router: say out loud when an application start attempt fails nxt_router_app_port_error() runs when a start attempt will never yield a port: the process that was asked for died before it could send one, or the message that would have carried it never arrived. The requests parked on that application are then answered 503 straight from nxt_router_app_start_failed(). The only trace of why was an nxt_debug(), which a default configuration does not emit -- so the observable behaviour was an application returning 503 with no line anywhere in the log to connect it to. Log it at NXT_LOG_ERR. It cannot be noisy: the handler is armed for one start attempt and disarmed as soon as the port arrives, so an ordinary worker exit or an idle timeout never reaches it at all. Two ways in do reach it without anything having gone wrong, and both are excluded rather than tolerated. The first is a restart. nxt_router_app_restart_handler() bumps app->generation and sends QUIT to the prototype, and a worker that prototype had already forked for an in-flight start goes with it. The REMOVE_PID for that worker still carries the start's stream -- the prototype set it when it forked, and nothing clears it for a process that never became ready -- so the router retypes it into an RPC error and it arrives here, reporting a start the operator themselves cancelled: an [error] line for a supported lifecycle operation working exactly as intended. Compare the generation, which is what nxt_router_app_port_ready() already does before it quietly QUITs a port that arrives for a superseded one. The second is a shutdown. nxt_port_close() runs nxt_port_rpc_close() over the router port, turning every registration still on it into an RPC error, so a start that was merely still pending when Unit was asked to stop would announce itself as a failure on the way out. Nothing bumps the generation there, so it needs its own test: engine->shutdown, which this file already consults in two other places before deciding whether to start another process at all. Only the log line is conditional in either case. The slots and the parked requests belong to an attempt that is over whatever ended it, so the cleanup runs unchanged. Every call that does log really is an application process that was asked for and will not exist. [error] rather than nxt_alert(). An alert is Unit's own fault; this one is usually the application's -- a module that fails to import, a worker that exits during its own startup -- and Unit is behaving exactly as designed when it reports one, so calling it an alert would be saying the wrong thing about whose problem it is. It would also break six tests that make an application fail on purpose, and, because the shared unitd's log window straddles test boundaries, intermittently break their neighbours too: test_python_application_loading_error, test_python_application_callable, test_asgi_application_loading_error, test_asgi_targets_broken and test_status_requests all take skip_alert and would need one more pattern; test_asgi_lifespan_failed takes none at all and would simply fail. [error] is emitted at the default log level, which is the whole point. A different sentence from the alert, too. nxt_alert() in nxt_router_start_app_process_handler() already says "app '%V' failed to start a process", and reusing it here would have been tidy and is a trap: Log.check_alerts() skips expected alerts with an unanchored re.search() over the log, so a test that quoted that sentence to skip this new [error] line would silently have skipped the genuine alert as well -- in itself, and in whatever ran beside it. The alert keeps the wording an operator may already be grepping for, and the new line gets its own. It still cannot say *why*. This is usually a REMOVE_PID that nxt_router_remove_pid_handler() retyped into an RPC error, and the body of a REMOVE_PID is the dead pid and nothing else -- the reason lives in whatever the dead process logged before exiting. That is the point of the line: it names the application and tells the operator there is something to go and look for. Co-Authored-By: Claude Opus 5 --- src/nxt_router.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 73ef49876..6d6ad31dc 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5124,6 +5124,7 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { uint32_t n; + nxt_bool_t restarted; nxt_app_t *app; nxt_app_joint_t *app_joint; nxt_app_joint_rpc_t *app_joint_rpc; @@ -5145,7 +5146,81 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, return; } - nxt_debug(task, "app '%V' %p start error", &app->name, app); + /* + * Two ways in are not failures at all, and both are excluded rather + * than tolerated. + * + * A restart: nxt_router_app_restart_handler() bumps app->generation + * and sends QUIT to the prototype, and a worker that prototype had + * already forked for an in-flight start goes with it. The REMOVE_PID + * for that worker still carries the start's stream -- the prototype + * set it when it forked (nxt_application.c) and nothing clears it for + * a process that never became ready -- so the router retypes it into + * an RPC error and it lands here, reporting a start the operator + * themselves cancelled. Nothing about the RPC peer is involved: the + * registration this handler belongs to never sets one. Generation is + * how nxt_router_app_port_ready() tells the same two apart before it + * quietly QUITs a port that arrives for a superseded one. + * + * And a shutdown: nxt_port_close() runs nxt_port_rpc_close() over the + * router port, which turns every registration still on it into an + * RPC error, so a start that was merely still pending when Unit was + * asked to stop would report itself as a failure. Nothing bumps the + * generation there, so it needs its own test -- engine->shutdown, the + * same flag this file already consults before deciding to start + * another process at all. + * + * Only the log line is conditional either way: the cleanup below has + * to run regardless, since the slots and the parked requests belong + * to an attempt that is over whatever ended it. + */ + + nxt_thread_mutex_lock(&app->mutex); + + restarted = (app->generation != app_joint_rpc->generation); + + nxt_thread_mutex_unlock(&app->mutex); + + /* + * Otherwise: logged, not nxt_debug()d. This handler is armed for one + * start attempt and disarmed the moment the port arrives, so it + * cannot fire for an ordinary worker exit or an idle timeout: every + * remaining call means an application process that was asked for will + * never exist, and the requests parked on it are about to be answered + * 503. At debug level that was invisible in a default configuration, + * which left "the application returns 503" with no corresponding line + * anywhere in the log. + * + * NXT_LOG_ERR rather than nxt_alert(), which the early-failure path + * in nxt_router_start_app_process_handler() uses. That one can only + * be Unit's own fault; this one is usually the application's -- a + * module that fails to import, a worker that exits during startup -- + * and Unit is working exactly as designed when it reports one. + * [error] is emitted at the default log level, which is the whole + * point of the change. + * + * A different sentence from that alert, deliberately, even though + * the two describe the same disappointment. The test suite skips + * expected alerts by an unanchored regex over the log + * (Log.check_alerts()), so a test that skipped this [error] line by + * quoting "failed to start a process" would silently swallow the + * alert as well, in every test that ran alongside it. The two are + * worth telling apart; the level alone is a thin thing to rely on. + * + * It cannot say *why*: this is usually a REMOVE_PID that + * nxt_router_remove_pid_handler() retyped, and its payload is the + * dead pid and nothing else. The reason is in whatever the dead + * process logged before exiting; this line is what tells the operator + * there is something to go and look for. + */ + + if (nxt_slow_path(restarted || task->thread->engine->shutdown)) { + nxt_debug(task, "app '%V' start attempt cancelled", &app->name); + + } else { + nxt_log(task, NXT_LOG_ERR, + "app '%V' start attempt produced no process", &app->name); + } if (app_joint_rpc->proto) { /* From fe2b1b99bd9d13cf9a29c64259b9421e94a5cfda Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 18:10:29 +0200 Subject: [PATCH 097/391] capability: stop handing unitd's capabilities to applications fork() copies every capability set verbatim, so whatever unitd holds, each application worker it starts holds too. For a unitd running as root that has never mattered: switching a child away from uid 0 clears the permitted, effective and ambient sets by itself. For a non-root unitd that was granted capabilities -- systemd "AmbientCapabilities=", file capabilities on the binary -- nothing clears them, in either of the two ways such a deployment can be arranged: - capget() filtered, the case the previous commit made survivable. setid stays clear, nxt_process_apply_creds() skips setgid()/setuid() altogether, and the sets are never touched. - capget() working. Credentials are switched, but from one nonzero uid to another, and the kernel only clears capabilities on a transition out of uid 0. So a unitd granted CAP_SETUID/CAP_SETGID to do its own job was giving those same capabilities to application code -- not because capget() was filtered, but because Unit never called capset() at all. The filtered case is merely the one where it is also invisible. Drop them in the child, at the tail of nxt_process_apply_creds(). That is the last point before a forked prototype, router, controller or discovery process does anything on an application's behalf, and it is after setgid()/setuid(), after unshare() and main's uid_map write, and after every mount, pivot_root and chroot nxt_proto_setup() performs for "rootfs" -- all of which need the capabilities being given up. Application workers are covered without running it: nxt_app_setup() calls init->start directly rather than going through apply_creds, so they inherit sets the prototype already emptied. Not in the main process, which is why this is a separate function rather than a line in nxt_capability_set(). Main binds listening sockets in nxt_main_listening_socket() every time the configuration changes, so it needs CAP_NET_BIND_SERVICE for as long as it runs; it also chown()s sockets, writes uid_map and moves prototypes into cgroups. Dropping there would replace "unitd fails to start" with "unitd starts but cannot listen on :80". The ambient set is cleared separately, first, with PR_CAP_AMBIENT_CLEAR_ALL. Clearing permitted clears ambient implicitly, but only if capset() succeeds, and ambient is the set that matters most: for a non-root process running a file with no capabilities, capabilities(7) reduces the execve() transformation to P'(permitted) = P'(effective) = P(ambient), so ambient is exactly what an "external" application -- Go, Node, anything reached through the execve() in nxt_external.c -- would otherwise carry into its own image. PR_CAP_AMBIENT_CLEAR_ALL needs no privilege and no prior read, so it works even where capget() and capset() are both denied. A plain #ifdef rather than an auto/ feature test: the macro either exists in or it does not, and a pre-4.3 kernel has no ambient set to clear and says EINVAL. The version passed to capset() is the compiled-in constant, not nxt_capability_linux_get_version(), because that probe is itself a capget() call and this has to work where capget() is what is filtered. _LINUX_CAPABILITY_VERSION_3 has been accepted since 2.6.26 and matches the two-element data array. The pid field is 0 rather than nxt_pid: capset() accepts only 0 or the caller's own pid, and nxt_pid holds the *global* pid in a process moved into a PID namespace, which would be EPERM. A denied capset() is not fatal in itself. Lowering all three sets to empty cannot be refused by the capability model -- every rule in capabilities(7) is a subset test, and the empty set is a subset of everything -- so an EPERM or ENOSYS means a syscall filter, which is an operator's decision, and this code runs in a forked child where an error is a dead worker the router keeps respawning rather than a visible startup failure. EINVAL and EFAULT still abort, because neither the version nor the pointer depends on anything outside this file. But warn-and-continue is the wrong answer on the path that matters. A prototype that cannot drop goes on to fork application workers holding unitd's capabilities -- exactly the outcome the drop exists to prevent, now reached silently under a log line that reads like housekeeping, and a hardened deployment produces precisely this shape, since a "SystemCallFilter=" allowlist that omits capget() rarely keeps capset(). So after a failed capset() the process re-reads what it still holds, and refuses the start if anything is left. An operator who filtered capset() and lost their application learns that from one alert naming the syscall; one who filtered it and kept a privileged application learns nothing. Router, controller and discovery take the warn-and-continue branch, where refusing would be a respawn loop; they hold nothing an application can reach. An application with "user": "root" keeps what root has. Dropping there would break compatibility and buy nothing, since a process with euid 0 regains a full permitted set on its next execve() of any ordinary file. This is also the escape hatch for the one case that does lose something: an application in a "credential" namespace begins with a full capability set in that namespace and, unless it is mapped to uid 0, now gives it up. Those capabilities were only valid inside the namespace, and Unit's own use of them is finished before this runs. CHANGES says so. The bounding set is left alone: emptying it needs CAP_SETPCAP, which the process being disarmed is not guaranteed to have. The tests assert which processes call capset() at all, which is the part that cannot pass by accident: the suite runs as an ordinary user with nothing to give up, so asserting an application holds no capabilities passes just as well without the code that empties them. The preload shim writes down the pid of every capset() caller and forwards the call untouched, so three claims about the design become assertions -- main never drops, the prototype does, the worker does not -- and it can fail that call on NXT_TEST_CAPSET_ERRNO, or fake a non-empty set on NXT_TEST_CAPGET_HOLD, to reach the refusal and its opposite. With the "still held" reader stubbed to answer unconditionally, both serves-anyway parametrisations fail with 503 while the refusal test still passes, which is what makes the guard load-bearing rather than decorative. Verified on Debian bookworm, amd64, over {EPERM, ENOSYS, no filter} x {no capabilities, ambient CAP_SETUID+CAP_SETGID} x {python, Go}: in every cell the application ends with CapInh, CapPrm, CapEff and CapAmb all zero, and in the capability-granted cells /proc//status shows unitd itself still held them. Before this commit the six capability-granted cells all report c0 in every set. Both syscall drift baselines gain capset, which is the point of the gate: it is a new kernel interface and the baseline is refreshed in the commit that introduced it. The musl leg also gains linux-headers, without which it was proving nothing here -- musl-dev does not carry , so auto/capability failed, NXT_HAVE_LINUX_CAPABILITY was off, and both nxt_capability_specific_set() and nxt_capability_drop() compiled to the stubs at the bottom of the file. That leg reported "no new syscalls" against a binary with no capability handling in it at all. With the header installed the musl capture gains capset and nothing else, which is also the confirmation that this is the only difference the missing header was hiding. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VXRyL2dx3yYBqaaYsdpuQE --- CHANGES | 40 +- docs/changes.xml | 42 +- src/nxt_capability.c | 634 ++++++++++++++++++++++++- src/nxt_capability.h | 9 + src/nxt_process.c | 70 ++- test/capget_filter.c | 132 ++++- test/syscalls/baseline-linux-glibc.txt | 1 + test/syscalls/baseline-linux-musl.txt | 1 + test/syscalls/run-in-docker.sh | 8 +- test/test_capget_fallback.py | 387 ++++++++++++++- 10 files changed, 1280 insertions(+), 44 deletions(-) diff --git a/CHANGES b/CHANGES index 7d6f86316..fb3dbfce3 100644 --- a/CHANGES +++ b/CHANGES @@ -8,8 +8,44 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 without assuming any capability: user and group switching and "rootfs" isolation are disabled for applications that do not enable the "credential" namespace, and applications run as unitd's own uid. - Capabilities the process may still hold are not dropped, since Unit - does not call capset(). Every other capget() failure remains fatal. + Every other capget() failure remains fatal. + + *) Security: application processes no longer inherit unitd's + capabilities. A unitd started as a non-root user that had been + granted capabilities -- systemd "AmbientCapabilities=", file + capabilities -- passed them through fork() to every application it + ran, because switching between two nonzero uids does not clear them + and a filtered capget() skips the switch entirely. Each forked + process now empties its permitted, effective, inheritable and + ambient sets once its credentials are final, before the worker + serves any request. The language module's own startup still runs + before the drop: the mount, pivot_root and chroot work for "rootfs" + isolation happens after it and needs the capabilities that are being + given up. Where a syscall filter denies capset() as well, an + application start is now refused rather than allowed to proceed with + capabilities still held: the request is answered 503 and the log + names the denied syscall. A process with an empty permitted, + effective and ambient set -- which is every process of a root unitd + once it has switched away from uid 0, and every process of an + unprivileged one that was granted no capabilities -- keeps behaving + as before, since there is nothing an application could have + inherited from it. Its inheritable set is emptied as well, which on + its own is not counted and grants nothing without an execve() of a + file carrying a matching capability. The + main process is unchanged, since it needs CAP_NET_BIND_SERVICE to + bind listening sockets on every reconfiguration; the router, + controller and discovery processes warn and carry on rather than + refuse, because they run no application code. Applications + configured with "user": "root" are also unchanged, since a root + process regains capabilities on the next execve() anyway. One + configuration does lose something it previously kept: an + application in a "credential" namespace began with a full + capability set inside that namespace, because the default uid_map + maps unitd's euid to the application's uid, so the setuid() is not + a transition out of uid 0 and nothing cleared it. Those + capabilities were valid only inside the namespace and Unit's own + use of them is finished before the drop, but an application that + relied on them now needs "user": "root". Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index b091589ec..635a48b89 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -18,9 +18,45 @@ seccomp profile -- made a non-root unitd fail to start with "failed to get process capabilities". Unit now warns and runs without assuming any capability: user and group switching and "rootfs" isolation are disabled for applications that do not enable the "credential" namespace, and -applications run as unitd's own uid. Capabilities the process may -still hold are not dropped, since Unit does not call capset(). Every -other capget() failure remains fatal. +applications run as unitd's own uid. Every other capget() failure +remains fatal. + + + + + +application processes no longer inherit unitd's capabilities. A unitd +started as a non-root user that had been granted capabilities -- systemd +"AmbientCapabilities=", file capabilities -- passed them through fork() +to every application it ran, because switching between two nonzero uids +does not clear them and a filtered capget() skips the switch entirely. +Each forked process now empties its permitted, effective, inheritable +and ambient sets once its credentials are final, before the worker +serves any request. The language module's own startup still runs before +the drop: the mount, pivot_root and chroot work for "rootfs" isolation +happens after it and needs the capabilities that are being given up. +Where a syscall filter denies capset() as well, an application start is +now refused rather than allowed to proceed with capabilities still held: +the request is answered 503 and the log names the denied syscall. A +process with an empty permitted, effective and ambient set -- which is +every process of a root unitd once it has switched away from uid 0, and +every process of an unprivileged one that was granted no capabilities -- +keeps behaving as before, since there is nothing an application could +have inherited from it. Its inheritable set is emptied as well, which +on its own is not counted and grants nothing without an execve() of a +file carrying a matching capability. The main process is +unchanged, since it needs CAP_NET_BIND_SERVICE to bind listening sockets +on every reconfiguration; the router, controller and discovery processes +warn and carry on rather than refuse, because they run no application +code. Applications configured with "user": "root" are also unchanged, +since a root process regains capabilities on the next execve() anyway. +One configuration does lose something it previously kept: an application +in a "credential" namespace began with a full capability set inside that +namespace, because the default uid_map maps unitd's euid to the +application's uid, so the setuid() is not a transition out of uid 0 and +nothing cleared it. Those capabilities were valid only inside the +namespace and Unit's own use of them is finished before the drop, but an +application that relied on them now needs "user": "root". diff --git a/src/nxt_capability.c b/src/nxt_capability.c index 16bdc7bc9..e341b3551 100644 --- a/src/nxt_capability.c +++ b/src/nxt_capability.c @@ -4,15 +4,21 @@ */ /* - * NOTE: this module currently exposes capability *detection* - * (capget), used by nxt_isolation.c and the credential machinery to - * decide whether a non-root build can honor setuid/setgid in the - * "user"/"group" config keys. Capabilities are NOT dropped - * programmatically via capset(2): the privilege barrier for app - * processes is the existing setuid + PR_SET_NO_NEW_PRIVS dance in - * nxt_credential.c / nxt_process.c. Operators expecting an explicit - * capability-drop step should not assume one exists; isolation in - * Unit relies on uid/namespace/seccomp boundaries, not capset. + * NOTE: this module does two separate things, in two different + * processes. + * + * nxt_capability_set() is *detection* (capget), run once in the main + * process from nxt_runtime_conf_init(). nxt_isolation.c and the + * credential machinery use its answer to decide whether a non-root + * unitd can honour setuid/setgid in the "user"/"group" config keys + * and "rootfs" isolation. + * + * nxt_capability_drop() is *disposal* (capset), run in every forked + * process from the tail of nxt_process_apply_creds(). It is never + * run in main, which keeps CAP_NET_BIND_SERVICE for the bind() in + * nxt_main_listening_socket() -- listeners are created on every + * reconfiguration, long after conf_init, so main cannot drop + * anything. See nxt_capability_drop() for the rest. */ #include @@ -20,7 +26,9 @@ #if (NXT_HAVE_LINUX_CAPABILITY) #include +#include #include +#include #if (_LINUX_CAPABILITY_VERSION_3) @@ -157,6 +165,607 @@ nxt_capability_specific_set(nxt_task_t *task, nxt_capabilities_t *cap) return NXT_OK; } + +/* + * The magic number rather than : one constant, stable + * since procfs existed, against a header that is not present on every + * libc Unit builds against. + */ +#define NXT_PROC_SUPER_MAGIC 0x9fa0 + + +static nxt_int_t +nxt_capability_status_line(u_char *p, u_char *end, uint64_t *bits) +{ + u_char *q; + uint64_t v; + nxt_uint_t digits; + + if (end - p <= (ssize_t) nxt_length("CapXxx:") + || memcmp(p, "Cap", 3) != 0) + { + return 0; + } + + if (memcmp(p, "CapPrm:", 7) != 0 + && memcmp(p, "CapEff:", 7) != 0 + && memcmp(p, "CapAmb:", 7) != 0) + { + /* CapInh and CapBnd, deliberately: see nxt_capability_still_held(). */ + return 0; + } + + q = p + nxt_length("CapXxx:"); + + while (q < end && (*q == ' ' || *q == '\t')) { + q++; + } + + v = 0; + digits = 0; + + while (q < end) { + if (*q >= '0' && *q <= '9') { + v = (v << 4) | (uint64_t) (*q - '0'); + + } else if (*q >= 'a' && *q <= 'f') { + v = (v << 4) | (uint64_t) (*q - 'a' + 10); + + } else if (*q >= 'A' && *q <= 'F') { + v = (v << 4) | (uint64_t) (*q - 'A' + 10); + + } else { + break; + } + + q++; + digits++; + } + + if (digits == 0 || digits > 16) { + return -1; + } + + *bits |= v; + + return 1; +} + + +static nxt_int_t +nxt_capability_proc_status_held(void) +{ + int fd; + u_char *p, *end, *eol; + size_t len; + ssize_t n; + uint64_t bits; + nxt_int_t ret; + nxt_uint_t found; + nxt_bool_t skipping; + struct statfs fs; + u_char buf[1024]; + + fd = open("/proc/self/status", O_RDONLY | O_CLOEXEC); + + if (fd < 0) { + return -1; + } + + /* + * Prove it is procfs before believing a word of it. An application + * that enables a "credential" namespace may also use "rootfs" even + * when capabilities are unknown, and nxt_proto_setup() has already + * pivot_root()ed into it by the time this runs -- with + * "automount": {"procfs": false} there may be no /proc there at all, + * or there may be a regular file of that name supplied with the + * application image. Reading capability sets out of one would let + * the image decide whether the prototype it is about to run keeps + * unitd's capabilities. + */ + + if (fstatfs(fd, &fs) != 0 || fs.f_type != NXT_PROC_SUPER_MAGIC) { + close(fd); + return -1; + } + + bits = 0; + found = 0; + len = 0; + skipping = 0; + + /* + * Streamed over a small buffer, with the tail of a partial line + * carried across reads, because this file has no bounded size: a + * process in enough supplementary groups pushes the Cap* lines past + * any fixed buffer on the strength of its "Groups:" line alone, and + * reading only the first N bytes would turn "holds nothing" into + * "cannot tell" and refuse it. A line too long for the buffer is + * skipped rather than grown into -- a Cap* line is 25 bytes. + */ + + for ( ;; ) { + n = read(fd, buf + len, sizeof(buf) - len); + + if (n < 0) { + + /* + * A signal that arrives mid-read is not an answer about + * capabilities. Without the retry it would become one: + * read() fails, the reader reports "cannot tell", and the + * caller refuses an application over a SIGCHLD. Every other + * errno is a real failure to read the file, and saying so is + * the honest reply. + */ + + if (nxt_errno == NXT_EINTR) { + continue; + } + + close(fd); + return -1; + } + + if (n == 0) { + break; + } + + end = buf + len + n; + p = buf; + + for ( ;; ) { + eol = memchr(p, '\n', (size_t) (end - p)); + + if (eol == NULL) { + break; + } + + if (!skipping) { + ret = nxt_capability_status_line(p, eol, &bits); + + if (nxt_slow_path(ret < 0)) { + close(fd); + return -1; + } + + found += ret; + } + + skipping = 0; + p = eol + 1; + } + + len = (size_t) (end - p); + + if (len == sizeof(buf)) { + skipping = 1; + len = 0; + + } else { + memmove(buf, p, len); + } + } + + /* + * Whatever is left when the file ends is a last line with no + * trailing newline. procfs always terminates the one it writes, so + * this is unreachable for the real /proc/self/status -- but the + * fstatfs() above is the only thing standing between this parser and + * a file supplied with an application image, and dropping the tail + * would silently turn a "CapPrm: 00000000000000c0" written without a + * newline into a file whose Cap* lines simply do not count. + */ + + if (len > 0 && !skipping) { + ret = nxt_capability_status_line(buf, buf + len, &bits); + + if (nxt_slow_path(ret < 0)) { + close(fd); + return -1; + } + + found += ret; + } + + close(fd); + + /* + * CapPrm and CapEff, at least. CapAmb needs Linux 4.3 and is + * allowed to be missing; fewer than two means this is not the file + * we think it is, and guessing from it would be worse than saying + * so. + */ + + if (found < 2) { + return -1; + } + + return bits != 0; +} + + +/* + * Does this process still hold anything an application could inherit? + * 1 yes, 0 no, -1 could not be determined. + * + * Consulted only after capset() has already been denied, to separate + * "the operator filtered a call that had nothing left to do" from "the + * sets really are still full". Without it the refusal in + * nxt_process_created_ok() would fire in the two commonest deployments + * there are, neither of which is unsafe: a root unitd, whose setuid() + * away from uid 0 empties permitted, effective and ambient before this + * function is ever reached, and an ordinary unprivileged unitd that was + * granted nothing. Both would lose every application to a filter that + * costs them nothing. + * + * Two readers are tried, cheapest first, because neither covers every + * deployment on its own. + * + * - capget(). A filter that denies capset() often still permits it, + * and this is the exact answer with no filesystem involved. It does + * not report the ambient set, which does not matter here: + * PR_CAP_AMBIENT_CLEAR_ALL in nxt_capability_drop() has already + * run, needs no privilege and cannot be refused by the capability + * model. pid 0 rather than nxt_pid for the same reason capset() + * uses it -- nxt_pid is the global pid, which does not resolve from + * inside a PID namespace. + * + * - /proc/self/status, when capget() is filtered too -- the shape this + * whole fallback exists for. A syscall filter cannot see through + * open() and read(). + * + * Both are readings. Nothing here is inferred from what this process + * ought to be holding, and an earlier draft that did -- a root unitd's + * setuid() away from uid 0 empties permitted, effective and ambient, so + * the transition alone looks like an answer -- was removed rather than + * patched. It was wrong two separate ways: SECBIT_KEEP_CAPS or + * SECBIT_NO_SETUID_FIXUP, either of which a service manager can set, + * suppress the clearing entirely; and inside a "credential" namespace a + * process can reach a nonzero euid while holding a full capability set + * in that namespace, never having made the transition at all. Both + * leave a process that looks disarmed and is not. It bought only the + * case where capget() is filtered *and* procfs is unreadable, and being + * wrong there is exactly where it costs the most. + * + * If neither reader answers, the caller refuses: this is a security + * fallback, and "could not tell" is not "there is nothing there". + * + * "Anything" means the permitted, effective and ambient sets, and only + * those three. The capset() this reader is consulted after still + * empties the inheritable set along with them whenever it is allowed to + * run -- what this function decides is narrower, and it is the only + * question worth asking here: is anything left that an application + * could use? + * + * An inheritable bit is not. It confers nothing in the process holding + * it; capabilities(7) turns it into privilege only on an execve() of a + * file whose own inheritable capability set intersects it. Unit's + * language modules are fork()ed and never exec()ed, so they cannot + * reach that rule at all, and an "external" application execs the + * binary named in its configuration -- an ordinary file, carrying no + * file capabilities unless an operator deliberately gave it some, in + * which case they asked for exactly this. Refusing over inheritable + * alone would therefore be refusing over something strictly narrower + * than the escalation this fallback exists to stop, and it is also + * precisely what an unprivileged unitd handed its workers before + * nxt_capability_drop() existed: nothing in Unit has ever cleared an + * inheritable set before now. + * + * That leaves a residual, and it is left deliberately: an "external" + * application whose own binary an operator gave a matching file + * inheritable capability does get that bit promoted on execve(). + * Refusing would not take it away -- capset() has just been denied, so + * nothing here can clear anything -- it would only take away the + * application. The trade is a certain outage on every host with a + * stray inheritable bit against a bit the operator had to place twice, + * once on unitd and once on the file it execs. + * + * Counting it would cost a great deal, too. Inheritable bits turn up on + * ordinary hosts without anyone meaning them to matter -- pam_cap exists + * to set exactly them, and the desktop this was measured on carries + * CapInh=0000000800000000 over entirely empty permitted, effective and + * ambient sets. A root unitd is no safer either, because setuid() away + * from uid 0 clears permitted, effective and ambient and leaves + * inheritable exactly as it was. Both would lose every application to a + * filter that costs them nothing. + * Conditioning it on PR_SET_NO_NEW_PRIVS does not rescue the idea + * either: no_new_privs is off in the default configuration -- + * nxt_isolation_main_prefork() sets isolation.new_privs to 1 and + * nxt_process_apply_creds() issues the prctl only when it is 0 -- so + * the condition would be false on exactly the hosts that have the + * stray bit. + * + * The bounding set is never counted either, for the reason it is never + * emptied: it is a ceiling on what an execve() may add, not something + * this process holds, and capset() would not have cleared it either. + * + * The ambient set is read where a reader offers it, even though the + * kernel keeps it a subset of permitted -- so an empty permitted + * already implies an empty ambient -- because the + * PR_CAP_AMBIENT_CLEAR_ALL in nxt_capability_drop() is only warned + * about when it fails. + */ +static nxt_int_t +nxt_capability_still_held(void) +{ + uint32_t bits; + struct __user_cap_data_struct data[2]; + struct __user_cap_header_struct hdr; + + nxt_memzero(data, sizeof(data)); + + hdr.version = NXT_CAPABILITY_VERSION; + hdr.pid = 0; + + if (nxt_fast_path(nxt_capget(&hdr, data) == 0)) { + bits = data[0].effective | data[0].permitted + | data[1].effective | data[1].permitted; + + return bits != 0; + } + + return nxt_capability_proc_status_held(); +} + + +/* + * Drop every capability the calling process still holds. Called from + * the tail of nxt_process_apply_creds(), which is the last thing that + * runs in a freshly forked prototype, router, controller or discovery + * process before it starts doing work on an application's behalf -- and + * after setgid()/setuid(), after unshare() and main's uid_map write, + * and after all the mount/pivot_root/chroot work nxt_proto_setup() does + * for "rootfs". Application workers never reach this function: the + * prototype forks them and nxt_app_setup() calls init->start directly, + * so they inherit sets this call already emptied. + * + * It is not the earliest possible point. nxt_proto_setup() calls the + * language module's setup first, and for PHP that is + * php_module_startup() with the php.ini named in the configuration, + * which loads and initialises whatever extensions that file lists. + * Nothing can be dropped before that: the mount, pivot_root and + * chroot calls later in the same function need CAP_SYS_ADMIN and + * CAP_SYS_CHROOT. The window is unchanged by this code and is not + * widened by it -- setgid()/setuid() have not happened yet either, so + * module startup has always run with unitd's own uid, which is the + * larger privilege of the two. + * + * Capabilities are per-thread, and capset() with pid 0 sets only the + * calling thread's, so this has to run while the process is still + * single-threaded -- and it does. The router's engine threads are + * created from init->start, which runs after this and inherits the + * emptied sets; the prototype and its workers never leave one thread; + * thread pools spawn on their first posted work item, which is later + * still. The one thread that could pre-exist is the signal thread, + * and only for an event engine that cannot poll signals itself + * (nxt_signal_thread_start(), reached from nxt_event_engine_create() + * when !interface->signal_support). Linux never selects one: epoll + * declares NXT_SIGNAL_EVENTS wherever signalfd exists, and rt->engine + * is only ever set to the interface Unit itself chose + * (nxt_runtime.c), never by an operator. Measured: with the router at + * nine threads, every /proc//task//status in every child + * reports empty sets. + * + * Why here and not in main: main binds listening sockets in + * nxt_main_listening_socket() every time the configuration changes, so + * it needs CAP_NET_BIND_SERVICE for the whole life of the process -- + * and also chown()s sockets, writes uid_map and moves children into + * cgroups. nxt_capability_set() runs from nxt_runtime_conf_init(), + * long before the first bind(), so a drop there would turn "unitd fails + * to start" into "unitd starts but cannot listen on :80". + * + * fork() copies the permitted, effective, inheritable, ambient and + * bounding sets verbatim, so without this a non-root unitd that was + * granted, say, CAP_SETUID/CAP_SETGID/CAP_SYS_CHROOT would hand those + * to every application worker. Two deployments reach that state: + * + * - capget() filtered. nxt_capability_specific_set() leaves setid + * clear, so nxt_process_apply_creds() skips setgid()/setuid() + * entirely and nothing lowers anything. + * - capget() working. The credentials are switched, but from one + * nonzero uid to another, and capabilities(7) only clears the sets + * on a transition *out of* uid 0 -- so the capabilities survive + * the setuid() as well. + * + * A root unitd is the case that was already safe: its setuid() away + * from 0 clears the permitted, effective and ambient sets by itself, + * which makes the capset() below a verified no-op there rather than a + * behaviour change. + * + * The bounding set is deliberately left alone: emptying it needs + * CAP_SETPCAP, which this process is about to give up and usually + * never had. It is a ceiling on what an execve() may add to the + * permitted set, and with inheritable emptied below the only thing + * left for it to admit is a file capability on the binary an + * application execve()s -- which is the operator's own choice, not + * something inherited from unitd. PR_SET_NO_NEW_PRIVS would close + * even that, but it is off in the default configuration (see the + * comment on nxt_capability_still_held() above), so it is not what + * makes this safe. + * + * The return value is three-way and advisory in the middle case: + * NXT_OK when nothing is left to inherit, NXT_ERROR when the call was + * malformed, and NXT_DECLINED when a syscall filter denied capset() + * and this process is still holding something as a result. + * NXT_DECLINED is deliberately not a refusal here. This function is + * reached by the router, the controller and discovery as well, through + * nxt_process_core_setup(), and failing those closed is worse than the + * bug it would be protecting against: router and controller carry + * .restart = 1 and are re-forked immediately and without backoff, so a + * filter that is going to deny every attempt turns into an unbounded + * respawn loop; and discovery notifies no one when it dies, leaving + * unitd running with no modules and no control socket. Only the caller + * knows whether application code is about to inherit what could not be + * dropped, so only the caller can decide. + */ +nxt_int_t +nxt_capability_drop(nxt_task_t *task) +{ + nxt_int_t held; + nxt_err_t err; + struct __user_cap_data_struct data[2]; + struct __user_cap_header_struct hdr; + + if (geteuid() == 0) { + /* + * An application configured with "user": "root" -- and the + * initial state of a root unitd, before apply_creds() has + * switched user -- keeps what root has. Dropping there would + * be a compatibility break with no security value: euid 0 + * regains a full permitted set on the next execve() of any + * ordinary file anyway (capabilities(7), "Capabilities and + * execution of programs by root"), so the drop could not hold + * for an exec'd application and would only make an embedded + * one inconsistently crippled. This mirrors the geteuid()==0 + * short-circuit in nxt_capability_set() above. + * + * It is also the escape hatch for the one case that does lose + * something here: an application in a "credential" namespace + * starts with a full capability set in that namespace + * (user_namespaces(7)), and unless it is mapped to uid 0 it + * now loses it. Those capabilities were only ever valid + * inside the namespace, and everything Unit itself does with + * them -- the mounts, pivot_root and chroot for "rootfs" -- + * has already happened in nxt_proto_setup() by the time this + * runs. An application that wants to keep them can ask for + * uid 0 within its own namespace. + * + * geteuid() rather than nxt_euid: nxt_euid is sampled once in + * nxt_lib_start() and is stale in a process that has just + * called setuid(). + */ + + return NXT_OK; + } + +#ifdef PR_CAP_AMBIENT + + /* + * Clear the ambient set first, and separately. For a non-root + * process executing a file with no capabilities, capabilities(7) + * reduces to P'(permitted) = P'(effective) = P(ambient): the + * ambient set is the only one that carries real privilege across + * execve(), inheritable survives but does nothing without a + * matching file inheritable set. So ambient is precisely what an + * "external" application -- Go, Node, anything Unit runs through + * the execve() in nxt_external.c -- would otherwise inherit. + * PR_CAP_AMBIENT_CLEAR_ALL needs no privilege and no prior read of + * the set, so it still works when capget() and capset() are both + * denied; emptying the permitted set below clears ambient + * implicitly, but only if that call succeeds. + * + * A plain #ifdef rather than an auto/ feature test: PR_CAP_AMBIENT + * is a macro in , so this compiles out on headers too + * old to define it, and a kernel older than 4.3 has no ambient set + * to clear and answers EINVAL, which is not worth a warning. + */ + + if (nxt_slow_path(prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_CLEAR_ALL, 0, 0, 0) + != 0)) + { + nxt_debug(task, "prctl(PR_CAP_AMBIENT_CLEAR_ALL) failed %E", + nxt_errno); + } + +#endif + + nxt_memzero(data, sizeof(data)); + + /* + * The compiled-in version constant, not + * nxt_capability_linux_get_version(): the probe is a capget() call, + * and this function has to work in the deployment where capget() is + * exactly what is filtered. On every + * header Unit builds against this is _LINUX_CAPABILITY_VERSION_3, + * which the kernel has accepted since 2.6.26 and which describes the + * two-element data array above. + * + * pid 0, not nxt_pid: capset() takes only 0 or the caller's own pid, + * and nxt_pid holds the *global* pid in a process that was moved into + * a PID namespace (nxt_process_whoami()), which would be EPERM here. + */ + + hdr.version = NXT_CAPABILITY_VERSION; + hdr.pid = 0; + + if (nxt_fast_path(nxt_capset(&hdr, data) == 0)) { + return NXT_OK; + } + + err = nxt_errno; + + /* + * Same errno split as nxt_capability_specific_set(): EPERM and + * ENOSYS are what a syscall filter returns, and a filter is an + * operator's decision, not an error. It is also the only way this + * call can fail with EPERM: lowering all three sets to empty is + * always permitted for any process (capabilities(7), the transition + * rules for a caller without CAP_SETPCAP), so a refusal cannot have + * come from the capability model itself. + * + * A denial is not automatically a problem, though, which is why + * nxt_capability_still_held() gets the last word: a root unitd has + * already emptied every set by switching away from uid 0, and an + * unprivileged one never held anything, so in both the call that + * was just refused would have done nothing. Only a process that + * demonstrably still holds something -- or cannot find out -- has + * anything to report. + * + * NXT_DECLINED rather than NXT_ERROR, and rather than NXT_OK: the + * consequence of a denied capset() is not the same for every + * caller, so this function reports "the sets are still full" and + * lets nxt_process_apply_creds()'s callers decide. A prototype + * refuses to start, because its whole remaining job is to fork + * application workers that would inherit those sets; the router, + * controller and discovery processes warn and continue, because + * they run no application code and killing them is worse than the + * bug -- see nxt_process_created_ok() and nxt_process_core_setup(). + * + * Anything else -- EINVAL, EFAULT -- means the call we just made is + * malformed, which can only be a bug in Unit, since neither the + * version nor the pointer depends on anything an operator controls. + * Fail closed there, for every caller alike. + */ + + if (err == NXT_EPERM || err == NXT_ENOSYS) { + held = nxt_capability_still_held(); + + if (held == 0) { + /* + * The overwhelmingly common case, and the reason this is + * measured rather than assumed. A root unitd has already + * cleared every set by switching away from uid 0, and an + * ordinary unprivileged unitd never had anything -- in both + * the call that was just denied would have been a no-op, and + * refusing over it would take the application down for no + * security benefit at all. Not even a warning: nothing + * happened. + */ + + nxt_debug(task, "capset() failed %E, and no capability is held", + err); + + return NXT_OK; + } + + if (held > 0) { + nxt_log(task, NXT_LOG_WARN, "capset() failed %E; the capabilities " + "this process holds could not be dropped", err); + + } else { + nxt_log(task, NXT_LOG_WARN, "capset() failed %E, and whether this " + "process holds any capability could not be determined " + "either", err); + } + + return NXT_DECLINED; + } + + nxt_alert(task, "failed to drop process capabilities: %E", err); + + return NXT_ERROR; +} + #else static nxt_int_t @@ -165,4 +774,11 @@ nxt_capability_specific_set(nxt_task_t *task, nxt_capabilities_t *cap) return NXT_OK; } + +nxt_int_t +nxt_capability_drop(nxt_task_t *task) +{ + return NXT_OK; +} + #endif diff --git a/src/nxt_capability.h b/src/nxt_capability.h index 030e6ccab..0364db179 100644 --- a/src/nxt_capability.h +++ b/src/nxt_capability.h @@ -15,5 +15,14 @@ typedef struct { NXT_EXPORT nxt_int_t nxt_capability_set(nxt_task_t *task, nxt_capabilities_t *cap); +/* + * NXT_OK: nothing is left to inherit -- either the sets were emptied, + * or they were already empty when a filter denied the attempt. + * NXT_DECLINED: capset() is filtered and this process is still holding + * capabilities, or cannot determine that it is not -- advisory, the + * caller decides whether that is fatal. NXT_ERROR: the call was + * malformed. + */ +NXT_EXPORT nxt_int_t nxt_capability_drop(nxt_task_t *task); #endif /* _NXT_CAPABILITY_INCLUDED_ */ diff --git a/src/nxt_process.c b/src/nxt_process.c index 831b7908f..2680f39e3 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -991,6 +991,39 @@ nxt_process_created_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) init = nxt_process_init(process); ret = nxt_process_apply_creds(task, process); + + if (nxt_slow_path(ret == NXT_DECLINED)) { + /* + * capset() is filtered and nxt_capability_still_held() found + * this process carrying capabilities because of it, or could + * not establish that it is not. Only a prototype reaches + * nxt_process_created_ok(): the core processes set + * NXT_PROCESS_STATE_READY in nxt_process_core_setup() and never + * send PROCESS_CREATED, and an application worker never sends + * one either, because nxt_app_setup() returns init->start() + * directly and that call does not come back with NXT_OK. The + * entire remaining job of a prototype is to fork workers that + * would inherit those sets. Refuse the start + * instead, and say which syscall and which consequence, so the + * operator's next step is to allow capset() rather than to + * hunt for a broken application. + * + * The refusal is visible: this exits nonzero, main's SIGCHLD + * reaper notifies the router with the start's stream still + * attached (main zeroes ->stream only at state READY, and its + * copy of a prototype that died here is still CREATED), the + * router turns that REMOVE_PID into an RPC error for the start + * attempt, and the requests waiting on the application are + * answered 503 rather than left to time out. + */ + + nxt_alert(task, "%s refused to start: capset() is denied, so the " + "capabilities this process holds cannot be dropped and " + "would be inherited by application code", process->name); + + goto fail; + } + if (nxt_slow_path(ret != NXT_OK)) { goto fail; } @@ -1039,7 +1072,24 @@ nxt_process_core_setup(nxt_task_t *task, nxt_process_t *process) nxt_int_t ret; ret = nxt_process_apply_creds(task, process); - if (nxt_slow_path(ret != NXT_OK)) { + + /* + * NXT_DECLINED -- capset() filtered, capabilities still held -- is + * warn-and-continue here, unlike on the nxt_process_created_ok() + * path. This is the router, the controller and discovery: they run + * no application code, so there is nothing to inherit what they + * keep, and refusing costs far more than it buys. Router and + * controller declare .restart = 1 and nxt_main_process_sigchld_ + * handler() re-forks them immediately and without backoff, so a + * filter that denies every attempt would spin a respawn loop + * instead of reporting anything; discovery's row in + * nxt_proc_remove_notify_matrix is all zeroes and it is the main + * process's only startup action, so its death would leave unitd up + * with no modules, no control socket and no error. + * nxt_capability_drop() has already logged the warning. + */ + + if (nxt_slow_path(ret != NXT_OK && ret != NXT_DECLINED)) { return NXT_ERROR; } @@ -1128,7 +1178,23 @@ nxt_process_apply_creds(nxt_task_t *task, nxt_process_t *process) } #endif - return NXT_OK; + /* + * Last: nothing after this point in any child needs a capability, + * and everything before it might. PR_SET_NO_NEW_PRIVS above stops + * this process *gaining* privilege across execve(); it does not + * take away what the process already carries, and fork() copies + * every capability set verbatim, so a unitd that was granted + * capabilities would otherwise pass them straight to application + * code. Main never calls apply_creds() and so never drops -- it + * binds listeners on every reconfiguration. + * + * Its NXT_DECLINED -- a filtered capset(), sets still full -- is + * passed through rather than folded into either NXT_OK or + * NXT_ERROR, because the two callers answer it differently: see + * nxt_process_created_ok() and nxt_process_core_setup(). + */ + + return nxt_capability_drop(task); } diff --git a/test/capget_filter.c b/test/capget_filter.c index a9ef04ab8..ac1f709b7 100644 --- a/test/capget_filter.c +++ b/test/capget_filter.c @@ -33,6 +33,41 @@ * no child could have written to proves nothing about what the * children did. * + * NXT_TEST_CAPSET_RECORD does the same for capset(2). That is how the + * test observes nxt_capability_drop(): the drop has no visible effect + * in a suite that runs without any capability to drop, so "the app has + * no capabilities" would pass just as well without the code. Which + * *processes* called capset() is the part that cannot pass by + * accident: main must never appear (it keeps CAP_NET_BIND_SERVICE for + * bind()), the prototype must, and the application worker must not, + * because it inherits the prototype's already-emptied sets rather than + * dropping again. + * + * NXT_TEST_CAPGET_HOLD is the other direction: a hex mask that makes + * capget(2) *succeed* and report that mask in the permitted, effective + * and inheritable sets of the first data element. The suite runs as an + * ordinary user with no capabilities, so without it there is nothing to + * refuse over -- nxt_capability_still_held() would correctly answer + * "empty" and the refusal below would never fire. Faking the answer is + * what lets an unprivileged pytest reach the code the harness smoke test + * reaches with real ambient capabilities. + * + * The mask the test passes is CAP_NET_BIND_SERVICE, deliberately: main's + * own capget() sees it too, and nxt_capability_specific_set() only looks + * for CAP_SETUID, CAP_SETGID and CAP_SYS_CHROOT, so unitd still starts + * and behaves exactly like the unprivileged process it really is. A + * mask containing CAP_SETUID would make it attempt a setuid() it cannot + * perform. + * + * NXT_TEST_CAPSET_ERRNO makes capset(2) fail the same way, which is + * the other half of a "SystemCallFilter=" allowlist: a profile that + * omits capget() almost always omits capset() too. The call is still + * recorded before it is failed, so the record says who *tried*. Unit + * answers that differently per process -- a prototype refuses to start + * rather than fork workers that would inherit what it could not drop, + * while the router, controller and discovery warn and carry on -- so + * the test needs to be able to produce it. + * * Build: * cc -shared -fPIC -o capget_filter.so capget_filter.c -ldl */ @@ -49,6 +84,8 @@ #include #include +#include + /* * glibc declares syscall() variadic, but it is an assembly stub that * only moves registers: it never walks a va_list, so calling it @@ -59,7 +96,10 @@ typedef long (*nxt_syscall_fn)(long, long, long, long, long, long, long); static nxt_syscall_fn nxt_real_syscall; static int nxt_capget_errno; /* 0: filter disabled */ +static int nxt_capset_errno; /* 0: filter disabled */ +static unsigned nxt_capget_hold; /* 0: report the real sets */ static char *nxt_record_path; /* strdup()ed, see above */ +static char *nxt_capset_record_path; static void @@ -75,11 +115,29 @@ nxt_shim_init(void) nxt_capget_errno = (int) strtol(value, NULL, 10); } + value = getenv("NXT_TEST_CAPSET_ERRNO"); + + if (value != NULL && *value != '\0') { + nxt_capset_errno = (int) strtol(value, NULL, 10); + } + + value = getenv("NXT_TEST_CAPGET_HOLD"); + + if (value != NULL && *value != '\0') { + nxt_capget_hold = (unsigned) strtoul(value, NULL, 16); + } + value = getenv("NXT_TEST_CAPGET_RECORD"); if (nxt_record_path == NULL && value != NULL && *value != '\0') { nxt_record_path = strdup(value); } + + value = getenv("NXT_TEST_CAPSET_RECORD"); + + if (nxt_capset_record_path == NULL && value != NULL && *value != '\0') { + nxt_capset_record_path = strdup(value); + } } @@ -92,12 +150,12 @@ nxt_shim_ctor(void) static void -nxt_shim_record(void) +nxt_shim_record(const char *path) { int fd, len; char buf[32]; - if (nxt_record_path == NULL) { + if (path == NULL) { return; } @@ -106,7 +164,7 @@ nxt_shim_record(void) * syscall(), so this cannot recurse back into the shim. */ - fd = open(nxt_record_path, O_WRONLY | O_CREAT | O_APPEND, 0644); + fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); if (fd < 0) { return; @@ -149,6 +207,36 @@ syscall(long number, ...) * Reading six arguments when the caller passed fewer yields * garbage in the unused ones, which the kernel ignores for every * syscall that does not declare them. + * + * ISO C calls that undefined (C17 7.16.1.1p2: va_arg with no + * actual next argument), and there is no portable fix -- a + * variadic forwarder cannot learn its own argument count, and + * re-declaring syscall() with a fixed prototype only trades this + * undefined behaviour for calling a function through an + * incompatible type. It is left as-is deliberately, because the + * concrete behaviour is the one every implementation already + * depends on: + * + * - glibc's syscall() is an assembly stub that unconditionally + * moves six argument registers, and musl's is a fixed + * seven-parameter C function. Neither ever walks a va_list, + * so both read exactly the slots this reads. + * - The slots read are always mapped memory, on every ABI Unit + * builds for. On x86-64 SysV the first five come from the + * register save area gcc/clang spill at entry and the sixth + * from the caller's own frame via overflow_arg_area; on + * aarch64 all seven fit the eight-register save area; on + * 32-bit ABIs the excess reads land in the caller's frame. + * None of them can fault, and none of them is used: only + * `number` decides what happens below. + * + * Whether the shim should install a real seccomp filter instead + * was weighed and rejected: a filter needs PR_SET_NO_NEW_PRIVS, + * after which it is inherited across both fork() and execve(), so + * every unitd child would see capget() fail too -- and the record + * file below, whose whole value is showing that only main was + * affected, could no longer say anything. It would also filter + * capset() in the children, which is the call the drop depends on. */ va_start(ap, number); @@ -162,10 +250,46 @@ syscall(long number, ...) #ifdef SYS_capget if (number == SYS_capget && nxt_capget_errno != 0) { - nxt_shim_record(); + nxt_shim_record(nxt_record_path); errno = nxt_capget_errno; return -1; } + + /* + * Only when a data pointer was passed: nxt_capability_linux_get_ + * version() calls capget() with NULL to have the kernel rewrite the + * header, and answering that one locally would leave the header + * untouched and change what Unit thinks the kernel supports. + */ + if (number == SYS_capget && nxt_capget_hold != 0 && (void *) a1 != NULL) { + struct __user_cap_data_struct *d = (void *) a1; + + memset(d, 0, 2 * sizeof(*d)); + + d[0].effective = nxt_capget_hold; + d[0].permitted = nxt_capget_hold; + d[0].inheritable = nxt_capget_hold; + + return 0; + } +#endif + +#ifdef SYS_capset + if (number == SYS_capset) { + /* + * Recorded first, then failed only if asked. The record is + * who *tried* to drop, which is what the unfiltered run + * asserts on; failing afterwards keeps that meaning intact + * for the filtered run, where the point is which processes + * survive a denial and which refuse to start. + */ + nxt_shim_record(nxt_capset_record_path); + + if (nxt_capset_errno != 0) { + errno = nxt_capset_errno; + return -1; + } + } #endif return nxt_real_syscall(number, a0, a1, a2, a3, a4, a5); diff --git a/test/syscalls/baseline-linux-glibc.txt b/test/syscalls/baseline-linux-glibc.txt index aca252e5b..bafcedfba 100644 --- a/test/syscalls/baseline-linux-glibc.txt +++ b/test/syscalls/baseline-linux-glibc.txt @@ -8,6 +8,7 @@ access arch_prctl bind brk +capset chmod clone clone3 diff --git a/test/syscalls/baseline-linux-musl.txt b/test/syscalls/baseline-linux-musl.txt index 680d3daec..9571b15c3 100644 --- a/test/syscalls/baseline-linux-musl.txt +++ b/test/syscalls/baseline-linux-musl.txt @@ -7,6 +7,7 @@ accept4 arch_prctl bind brk +capset chmod clone close diff --git a/test/syscalls/run-in-docker.sh b/test/syscalls/run-in-docker.sh index 235c83e73..aee882022 100755 --- a/test/syscalls/run-in-docker.sh +++ b/test/syscalls/run-in-docker.sh @@ -57,7 +57,13 @@ if [ "$LIBC" = glibc ]; then gcc make libc6-dev libpcre2-dev strace curl ca-certificates >/dev/null' else IMAGE=alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce - INSTALL='apk add --no-cache gcc make musl-dev pcre2-dev strace curl >/dev/null' + # linux-headers, which musl-dev does not carry: is + # what auto/capability probes for, and without it NXT_HAVE_LINUX_CAPABILITY + # is off, nxt_capability_specific_set() and nxt_capability_drop() compile + # to stubs, and this leg reports "no new syscalls" about a binary that has + # no capability handling in it at all. + INSTALL='apk add --no-cache gcc make musl-dev linux-headers pcre2-dev \ + strace curl >/dev/null' fi command -v docker >/dev/null 2>&1 || die "docker not found" diff --git a/test/test_capget_fallback.py b/test/test_capget_fallback.py index 841d6a9e1..e4b505e56 100644 --- a/test/test_capget_fallback.py +++ b/test/test_capget_fallback.py @@ -8,6 +8,48 @@ aborts, because a privilege downgrade caused by a bug in Unit is worse than a hard failure. +The same shim also watches capset(2), which is how this file covers +nxt_capability_drop() -- the drop every forked process performs at the +tail of nxt_process_apply_creds(). Its effect cannot be tested here: +the suite runs as an ordinary user with no capabilities, so "the +application has none either" is true with or without the code. What +can be tested, and cannot pass by accident, is *which processes drop*. +Main must not, because it binds listening sockets on every +reconfiguration; the prototype must, because it is the ancestor of +every worker; and the worker itself must not, because it inherits. +A run with real capabilities to lose needs root, and lives in the +freeunit-harness smoke test instead. + +The shim can deny capset(2) as well, which is the deployment a +"SystemCallFilter=" allowlist actually produces -- a profile that omits +capget() rarely keeps capset(). Unit answers that per process: the +prototype refuses to start rather than fork workers that would inherit +what it could not drop, while the router, controller and discovery warn +and carry on, because they run no application code and killing them +would be a respawn loop or a silent death. Both halves are asserted in +test_capset_filtered_refuses_the_application(). + +The refusal is conditional on there being something to refuse over, and +that is the harder half to test here, because a suite running as an +ordinary user holds nothing. So the shim can also make capget() report +a set it does not have -- which is what lets the refusal be reached at +all -- and the opposite case, a denied capset() with genuinely empty +sets, gets its own test: it must serve, through either of the two +readers nxt_capability_still_held() can answer from. + +One branch stays out of reach here, and is named rather than left to be +found: the /proc/self/status reader reporting a *non-empty* set. The +refusal tests get their non-empty answer from the shim's fake capget(), +because the procfs reader reads the kernel and the kernel will not lie +for a suite that holds nothing; faking it would mean interposing open(), +read() and fstatfs() as well, and that fstatfs() check exists precisely +to stop a file being substituted for the real one. What is covered +unprivileged is that reader answering "empty" rather than "cannot tell" +-- the parsing, the streaming and the fstatfs() check, all of which fail +closed into a refusal if they break. Only the final comparison needs a +run with something genuinely held, under a filter that denies capget() +and capset() together, which is the harness smoke test's ground. + Reproducing a seccomp filter in a test would need privileges the suite does not have, so the errno is injected instead: src/nxt_capability.c calls the kernel through the nxt_capget() macro, which expands to @@ -52,6 +94,7 @@ import re import shutil import signal +import stat import subprocess import tempfile import time @@ -68,6 +111,31 @@ WARNING = r'\[warn\].+capget\(\) failed.+capabilities are unknown' ALERT = r'\[alert\].+failed to get process capabilities' +# nxt_capability_drop() when capset(2) is denied. Deliberately matched on +# the syscall alone and not on the sentence around it: this is the test's +# proof that the injected failure reached Unit at all, and it has to hold on +# both sides of the change it is a control for. Wording that only the fixed +# build emits would make the control fail for the wrong reason. +CAPSET_WARNING = r'\[warn\].+capset\(\) failed' + +# The two lines the refusal writes: one from the prototype that would have +# forked the workers, one from the router that was waiting for it. +REFUSED = r'\[alert\].+refused to start: capset\(\) is denied' +START_FAILED = r"\[error\].+app '.+' start attempt produced no process" + +# nxt_capability_still_held() giving up -- neither capget() nor +# /proc/self/status could say what the process holds. Never expected in +# this suite; asserted absent so that a test which reaches the "serves" +# outcome cannot do so by way of a reader that silently broke. +UNDETERMINED = r'capset\(\) failed.+could not be determined' + +# CAP_NET_BIND_SERVICE. Handed to the shim so capget() reports a +# non-empty set: the suite runs with no capabilities of its own, so +# without this there is genuinely nothing to refuse over. This +# particular bit because nxt_capability_specific_set() ignores it -- +# unitd goes on behaving as the unprivileged process it really is. +HELD = 1 << 10 + # The two errnos a syscall filter conventionally returns for a denied # call. nxt_capability.c accepts both, so both belong in the test. FILTERED = [errno.EPERM, errno.ENOSYS] @@ -81,6 +149,31 @@ def requirements(): if option.is_privileged: pytest.skip('a root unitd never calls capget()') + # Everything below reaches unitd through LD_PRELOAD, and everything + # below reasons about capabilities this process can see. A set-id bit + # or a file capability on the binary breaks both at once: glibc treats + # the exec as secure and drops LD_PRELOAD, so the shim never arms, and + # the spawned unitd gets capabilities the pytest process does not have, + # which would make the "nothing is held" premise a statement about the + # wrong process. Neither is true of an ordinary build, but "ordinary" + # is the sort of thing worth checking rather than assuming. + unitd = Path(f'{option.current_dir}/build/sbin/unitd') + + if unitd.is_file(): + if unitd.stat().st_mode & (stat.S_ISUID | stat.S_ISGID): + pytest.skip('unitd is set-id; LD_PRELOAD would be dropped') + + try: + os.getxattr(unitd, 'security.capability') + + except OSError: + pass # The ordinary case: no file capabilities. + + else: + pytest.skip( + 'unitd has file capabilities; LD_PRELOAD would be dropped' + ) + @pytest.fixture(scope='module') def shim(requirements): @@ -123,7 +216,7 @@ class Unitd: """A unitd of our own, since conftest's shared one has no way to take an environment.""" - def __init__(self, shim, errno): + def __init__(self, shim, errno, capset_errno=0, capget_hold=0): builddir = f'{option.current_dir}/build' self.unitd = f'{builddir}/sbin/unitd' @@ -138,6 +231,7 @@ def __init__(self, shim, errno): self.log = f'{self.dir}/unit.log' self.stderr = f'{self.dir}/stderr.log' self.record = f'{self.dir}/capget.pids' + self.capset_record = f'{self.dir}/capset.pids' self.control = f'{self.dir}/control.unit.sock' self.listener = f'{self.dir}/app.sock' self.pidfile = f'{self.dir}/unit.pid' @@ -153,6 +247,15 @@ def __init__(self, shim, errno): env['LD_PRELOAD'] = shim env['NXT_TEST_CAPGET_ERRNO'] = str(errno) env['NXT_TEST_CAPGET_RECORD'] = self.record + env['NXT_TEST_CAPSET_RECORD'] = self.capset_record + + # Only when asked, so the tests that do not deny capset() run + # with exactly the environment they ran with before. + if capset_errno: + env['NXT_TEST_CAPSET_ERRNO'] = str(capset_errno) + + if capget_hold: + env['NXT_TEST_CAPGET_HOLD'] = f'{capget_hold:x}' # unitd's stderr is kept apart from unit.log on purpose. The # first warning is written from nxt_runtime_conf_init(), before @@ -201,18 +304,24 @@ def read(self, path): return Path(path).read_text(encoding='utf-8', errors='ignore') def filtered_pids(self): - if not Path(self.record).exists(): + return self.recorded_pids(self.record) + + def capset_pids(self): + return self.recorded_pids(self.capset_record) + + def recorded_pids(self, path): + if not Path(path).exists(): return set() - return {int(p) for p in self.read(self.record).split()} + return {int(p) for p in self.read(path).split()} @pytest.fixture() def unitd_factory(shim): started = [] - def factory(errno): - instance = Unitd(shim, errno) + def factory(errno, capset_errno=0, capget_hold=0): + instance = Unitd(shim, errno, capset_errno, capget_hold) started.append(instance) return instance @@ -262,11 +371,37 @@ def app_conf(appdir): pytest.skip('requires the python or php module') +def caps_of(pid='self'): + """The four capability sets of a process, as integers. + + CapAmb needs Linux 4.3 and CapInh has been there forever, but read + both defensively: a set the file does not name reads as empty, which + is the answer that cannot make a test pass by accident here. + """ + status = Path(f'/proc/{pid}/status').read_text(encoding='utf-8') + caps = {} + + for name in ('CapInh', 'CapPrm', 'CapEff', 'CapAmb'): + found = re.search(rf'^{name}:\s+([0-9a-f]+)', status, re.M) + caps[name] = int(found.group(1), 16) if found is not None else 0 + + return caps + + +def count_matches(path, pattern): + if not Path(path).exists(): + return 0 + + text = Path(path).read_text(encoding='utf-8', errors='ignore') + + return len(re.findall(pattern, text, re.M)) + + def wait_for(path, pattern, timeout=150): for _ in range(timeout): if Path(path).exists(): - found = re.search(pattern, Path(path).read_text( - encoding='utf-8', errors='ignore'), re.M) + text = Path(path).read_text(encoding='utf-8', errors='ignore') + found = re.search(pattern, text, re.M) if found is not None: return found @@ -292,21 +427,7 @@ def test_capget_filtered_keeps_running(shim, unitd_factory, filtered_errno): assert wait_for(unitd.log, WARNING) is not None, 'warn in unit.log' assert wait_for(unitd.log, r'router started') is not None, 'router started' - appdir = f'{unitd.dir}/app' - Path(appdir).mkdir() - conf = { - "listeners": {f'unix:{unitd.listener}': {"pass": "applications/app"}}, - "applications": {"app": app_conf(appdir)}, - } - public_dir(appdir) - - resp = client.put( - url='/config', - sock_type='unix', - addr=unitd.control, - body=json.dumps(conf), - ) - assert 'success' in resp['body'], 'configure' + assert 'success' in configure_app(unitd)['body'], 'configure' resp = client.get(sock_type='unix', addr=unitd.listener) assert resp['status'] == 200, 'application answers' @@ -320,6 +441,40 @@ def test_capget_filtered_keeps_running(shim, unitd_factory, filtered_errno): assert uid is not None, 'app Uid' assert int(uid.group(2)) == os.geteuid(), 'app runs as unitd uid' + # The application holds no capabilities at all -- inheritable + # included, which is the one set nxt_capability_still_held() does not + # count but nxt_capability_drop() does still empty. On a suite that + # runs without any to begin with the other three cannot fail, which + # is why they are not the real assertion; see the capset() + # bookkeeping below. CapInh is not in that position: this host may + # well have one, and the drop is what clears it. + for name, value in caps_of(app_pid).items(): + assert value == 0, f'app {name}' + + # The real assertion: nxt_capability_drop() ran, and ran in exactly + # the processes it is supposed to. Unlike the zeroes above, none of + # this can pass by accident -- on the commit before this one the + # file does not exist at all. + main_pid = int(unitd.read(unitd.pidfile).strip()) + proto_pid = int( + re.search(r'^PPid:\s+(\d+)', status, re.M).group(1) + ) + dropped = unitd.capset_pids() + + # Main must never drop: it binds listening sockets in + # nxt_main_listening_socket() on every reconfiguration and so needs + # CAP_NET_BIND_SERVICE for as long as it lives. + assert main_pid not in dropped, 'main keeps its capabilities' + + # The prototype must, since it is the ancestor of every worker. + assert proto_pid in dropped, 'the prototype drops' + + # The worker must not: nxt_app_setup() calls init->start directly + # without going through nxt_process_apply_creds(), so it inherits + # sets the prototype already emptied. If that ever changes this + # assertion is the thing that says so. + assert app_pid not in dropped, 'the worker inherits rather than drops' + # The application inherited the shim -- fork() carries the mapping, # armed, whatever the environment says afterwards ... maps = Path(f'/proc/{app_pid}/maps').read_text(encoding='utf-8') @@ -328,7 +483,6 @@ def test_capget_filtered_keeps_running(shim, unitd_factory, filtered_errno): # ... and still only the main process ever had a capget() filtered, # which is what makes the inheritance harmless. Two entries: the # version probe and the call that matters. - main_pid = int(unitd.read(unitd.pidfile).strip()) assert unitd.filtered_pids() == {main_pid}, 'only unitd asked' @@ -342,3 +496,190 @@ def test_capget_einval_still_fatal(unitd_factory): # stderr is the only place this can be recorded. assert not Path(unitd.log).exists(), 'no unit.log' assert re.search(ALERT, unitd.read(unitd.stderr), re.M), 'alert on stderr' + + +def configure_app(unitd): + """Load a one-process application and return the config response.""" + appdir = f'{unitd.dir}/app' + Path(appdir).mkdir() + public_dir(appdir) + + conf = { + "listeners": {f'unix:{unitd.listener}': {"pass": "applications/app"}}, + "applications": {"app": app_conf(appdir)}, + } + + return client.put( + url='/config', + sock_type='unix', + addr=unitd.control, + body=json.dumps(conf), + ) + + +@pytest.mark.parametrize('filtered_errno', FILTERED, ids=errno.errorcode.get) +def test_capset_filtered_refuses_the_application( + shim, unitd_factory, filtered_errno +): + """A denied capset() with capabilities still held must refuse. + + This is the deployment the whole file is about, taken one step + further: a "SystemCallFilter=" allowlist that omits capget() rarely + keeps capset(), so the fallback that lets unitd start lands in a + prototype that then cannot give up what it holds. Warning and + continuing there would hand unitd's capabilities to application + code -- silently, since the operator asked for neither. + + The refusal is deliberately not made inside nxt_capability_drop(). + That function is also on the router's, the controller's and + discovery's path through nxt_process_core_setup(), and those three + run no application code: router and controller are re-forked + immediately and without backoff, so refusing there would be a + respawn loop, and discovery notifies nobody when it dies. Both + halves are asserted below -- unitd comes up and stays up, and only + the application start fails. + """ + unitd = unitd_factory(0, capset_errno=filtered_errno, capget_hold=HELD) + + assert waitforfiles(unitd.control), ( + f'unitd started; its stderr was:\n{unitd.read(unitd.stderr)}' + ) + + # The injected failure reached Unit, and nxt_capability_still_held() + # found something to refuse over -- without both, the rest of this + # test could pass against a shim that never fired. And the core + # processes met the same denial in nxt_process_core_setup() and + # carried on: the controller is answering on the socket above and the + # router logged its own start line. + assert wait_for(unitd.log, CAPSET_WARNING) is not None, 'capset denied' + assert wait_for(unitd.log, r'router started') is not None, 'router started' + + assert 'success' in configure_app(unitd)['body'], 'configure' + + # The application does not serve. 503 rather than a hang: the + # prototype exits, main's SIGCHLD reaper notifies the router with the + # start's stream still attached, and the router fails the requests + # parked on the application instead of letting them time out. + resp = client.get(sock_type='unix', addr=unitd.listener) + assert resp['status'] == 503, 'application refused' + + # Both ends of that route say so above debug level, which is what an + # operator sees in a default configuration: the prototype names the + # syscall and the consequence, the router names the application. + assert wait_for(unitd.log, REFUSED) is not None, 'prototype refuses' + assert wait_for(unitd.log, START_FAILED) is not None, 'router reports' + + # No respawn loop. Nothing in Unit re-arms a start on its own -- + # every nxt_router_start_app_process() call site is driven by a + # request, a port becoming ready or a configuration change -- so + # after the one attempt above the count must stay where it is. + refusals = count_matches(unitd.log, REFUSED) + assert refusals == 1, 'exactly one attempt' + + time.sleep(2) + assert count_matches(unitd.log, REFUSED) == refusals, 'no retry on its own' + + # unitd itself is unharmed, which is the half that would have been + # lost by refusing inside nxt_capability_drop(). + assert unitd.process.poll() is None, 'unitd still running' + resp = client.get(url='/config', sock_type='unix', addr=unitd.control) + assert resp['status'] == 200, 'controller still answering' + + # The failure is stable rather than latched: a second request makes a + # second attempt, and that attempt fails the same way. Only the 503 + # is asserted -- whether Unit remembers the refusal or repeats it is + # an implementation choice this test does not fix. + resp = client.get(sock_type='unix', addr=unitd.listener) + assert resp['status'] == 503, 'still refused' + + # Main must still never appear among the processes that tried to + # drop, filter or no filter. + main_pid = int(unitd.read(unitd.pidfile).strip()) + tried = unitd.capset_pids() + assert tried, 'somebody tried to drop' + assert main_pid not in tried, 'main never tries' + + +# Which reader nxt_capability_still_held() has to answer from: capget() +# when the filter left it alone, /proc/self/status when it did not. +READERS = {'capget': 0, 'proc': errno.EPERM} + + +@pytest.mark.parametrize('reader', READERS) +def test_capset_filtered_serves_when_nothing_is_held( + shim, unitd_factory, reader +): + """A denied capset() with nothing to drop must not refuse anything. + + The refusal above is conditional on actually holding something, and + that condition is not decoration. Both of the commonest deployments + reach nxt_capability_drop() with every set already empty: a root + unitd, because switching away from uid 0 clears them, and an + ordinary unprivileged unitd, because it was never granted anything. + A filter that denies capset() in either of those denies a call that + had nothing left to do, and refusing over it would take every + application down for no security benefit whatsoever. + + Both readers are exercised, because they cover different filters and + a broken one fails the same way in each: silently, by returning "I + cannot tell", which is treated as a refusal. That is why the + absence of the "could not be determined" warning is asserted rather + than assumed -- without it this test would still pass with both + readers removed, on the strength of a refusal that never happened. + """ + # The premise, stated rather than inherited from whatever the host + # happens to be. unitd is about to be forked from this process, so + # "nothing is held" is a property of this process, and the reader is + # going to read it back through whichever of the two legs is under + # test. Without this the test says only "the host agreed with us + # today": on a runner with empty sets it passes for the right reason, + # and on a host that does hold something it would fail with a 503 and + # no clue as to why. + caps = caps_of() + held = { + name: caps[name] + for name in ('CapPrm', 'CapEff', 'CapAmb') + if caps[name] != 0 + } + + if held: + pytest.skip(f'the test process holds capabilities: {held}') + + # CapInh is deliberately not in that list, and this is the test that + # says so. An inheritable bit confers nothing without an execve() of + # a file carrying a matching one, so nxt_capability_still_held() does + # not count it -- and a desktop with pam_cap or a container runtime in + # the picture commonly has one, which is exactly the host where + # counting it would refuse every application for nothing. This run is + # only evidence of that when the bit is actually set, so record which + # kind of host produced the result. + print(f'\nCapInh of the test process: {caps["CapInh"]:016x}') + + unitd = unitd_factory(READERS[reader], capset_errno=errno.EPERM) + + assert waitforfiles(unitd.control), ( + f'unitd started; its stderr was:\n{unitd.read(unitd.stderr)}' + ) + assert wait_for(unitd.log, r'router started') is not None, 'router started' + + assert 'success' in configure_app(unitd)['body'], 'configure' + + resp = client.get(sock_type='unix', addr=unitd.listener) + assert resp['status'] == 200, 'application serves' + + log = unitd.read(unitd.log) + assert re.search(UNDETERMINED, log, re.M) is None, 'the reader answered' + assert re.search(REFUSED, log, re.M) is None, 'nothing was refused' + + # And the drop still happened where it always does, so the reader is + # not being consulted in place of the capset() itself. + proto_pid = int( + re.search( + r'^PPid:\s+(\d+)', + Path(f'/proc/{int(resp["body"].strip())}/status').read_text( + encoding='utf-8' + ), + re.M, + ).group(1) + ) + assert proto_pid in unitd.capset_pids(), 'the prototype still tried' From 3da7bb99b2702e98b8f3e794187aac41e3197fbe Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 19:04:40 +0200 Subject: [PATCH 098/391] http: review follow-ups to the bodyless upstream response All mechanical, no behaviour change; #295 merged before these landed. nxt_http_request_is_bodyless() is static again and takes no status parameter. It has one caller, in its own translation unit, and only ever asks about r->status. Only nxt_http_request_is_bodyless_final(), which the h1 peer reader calls with peer->status before r->status exists, needs external linkage, so nxt_http.h declares just that one. nxt_http_proxy_buf_mem_hold() registers its pool cleanup with &r->task rather than the task current at the call. nxt_mp_cleanup() stores that pointer in the work item (src/nxt_mp.c:1081) and hands it back when nxt_mp_destroy() runs the cleanup, and the task passed there belongs to the upstream connection. That connection is closed from nxt_http_proxy_send_body() before the pool's final release, and its nxt_conn_t goes back to the engine freelist to be re-zeroed and reused -- so the stored pointer is left addressing recycled memory. Both frees are work-queue-deferred, so the ordering is the normal one rather than a guaranteed one, and the pointer stayed harmless only because the handler never dereferences it: a latent hazard, not a live use-after-free. &r->task lives in the pool being destroyed, and nxt_mp_destroy() runs its cleanups before freeing any block, so it is valid for exactly as long as the handler can run. The comment says why the task is passed at all, so that the next reader does not reach for it. The port registry row for 7978 was a cell short of the table's five columns, so test_proxy_head.py rendered under "Rust handler". The test module docstring named two of the three paths an unrelayed upstream header buffer can take. The third -- an ordinary proxied response whose body arrives in a read after the header, which stranded the request pool on master as well -- is now named there, with the same caveat that it is not observable from the client. Two comments in that file still described a revision that #295 superseded. The bodyless branch has one exit, not two: it holds the buffer whether or not bytes followed the header, so /cl10junk does not take "a different exit" from /cl10, /204 and /304 -- it reaches the same one with bytes still in the buffer. What the cases contrast is the shape of the header read, and both comments now say that. The 7978 registry row also claimed "header block only", while the port serves /cl10junk (header plus trailing bytes) and /ok (a real body). Refs #295 Co-Authored-By: Claude Opus 5 --- src/nxt_http.h | 2 -- src/nxt_http_proxy.c | 18 +++++++++++++----- src/nxt_http_request.c | 13 +++++++------ test/fake_upstream/README.md | 2 +- test/test_proxy_head.py | 26 ++++++++++++++++---------- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/nxt_http.h b/src/nxt_http.h index 49affb288..5571fd31c 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -431,8 +431,6 @@ void nxt_http_request_ws_frame_start(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *ws_frame); void nxt_http_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out); -nxt_bool_t nxt_http_request_is_bodyless(nxt_http_request_t *r, - nxt_http_status_t status); nxt_bool_t nxt_http_request_is_bodyless_final(nxt_http_request_t *r, nxt_http_status_t status); nxt_buf_t *nxt_http_buf_mem(nxt_task_t *task, nxt_http_request_t *r, diff --git a/src/nxt_http_proxy.c b/src/nxt_http_proxy.c index db40f2a74..d3b36a11e 100644 --- a/src/nxt_http_proxy.c +++ b/src/nxt_http_proxy.c @@ -384,9 +384,9 @@ nxt_http_proxy_buf_mem_cleanup(nxt_task_t *task, void *obj, void *data) engine = data; /* - * Runs from nxt_mp_destroy(), which may be reached from a different task - * than the one current when the buffer was held, so "task" is not touched - * and the engine is carried in "data" instead. + * The engine is carried in "data": nxt_mp_destroy() has none to hand. + * "task" is passed only so the pointer nxt_mp_cleanup() stores stays + * inside the pool being destroyed -- do not dereference it here. */ nxt_event_engine_buf_mem_free(engine, b); @@ -426,8 +426,16 @@ nxt_http_proxy_buf_mem_hold(nxt_task_t *task, nxt_http_request_t *r, { nxt_int_t ret; - ret = nxt_mp_cleanup(r->mem_pool, nxt_http_proxy_buf_mem_cleanup, task, b, - task->thread->engine); + /* + * &r->task, not "task": nxt_mp_cleanup() stores the task pointer in the + * work item and hands it back at destroy time, and the peer connection's + * task is freed with that connection well before the request pool. + * &r->task lives in the pool being destroyed, which nxt_mp_destroy() runs + * its cleanups before freeing. + */ + + ret = nxt_mp_cleanup(r->mem_pool, nxt_http_proxy_buf_mem_cleanup, &r->task, + b, task->thread->engine); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; } diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index ae5ed8d92..d20af82ef 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -22,6 +22,7 @@ static void nxt_http_request_forward_protocol(nxt_http_request_t *r, static void nxt_http_request_ready(nxt_task_t *task, void *obj, void *data); static void nxt_http_request_proto_info(nxt_task_t *task, nxt_http_request_t *r); +static nxt_bool_t nxt_http_request_is_bodyless(nxt_http_request_t *r); static void nxt_http_request_drop_framing_fields(nxt_http_request_t *r); static nxt_buf_t *nxt_http_request_body_drop(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out); @@ -706,7 +707,7 @@ nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, * verbatim and unframed, which a downstream parser reads as the start of * the next response. */ - r->no_body = nxt_http_request_is_bodyless(r, r->status); + r->no_body = nxt_http_request_is_bodyless(r); /* * RFC 9110 Sect. 8.6: a server must not send Content-Length in a 1xx or @@ -846,8 +847,8 @@ nxt_http_request_is_bodyless_final(nxt_http_request_t *r, } -nxt_bool_t -nxt_http_request_is_bodyless(nxt_http_request_t *r, nxt_http_status_t status) +static nxt_bool_t +nxt_http_request_is_bodyless(nxt_http_request_t *r) { /* * A 101 upgrade is a 1xx status, but the bytes that follow its header are @@ -861,15 +862,15 @@ nxt_http_request_is_bodyless(nxt_http_request_t *r, nxt_http_status_t status) * alone would leave an application that answers a WebSocket-upgrade * request with 204 plus a body on the unframed path. */ - if (r->websocket_handshake && status == NXT_HTTP_SWITCHING_PROTOCOLS) { + if (r->websocket_handshake && r->status == NXT_HTTP_SWITCHING_PROTOCOLS) { return 0; } - if (status >= NXT_HTTP_CONTINUE && status < NXT_HTTP_OK) { + if (r->status >= NXT_HTTP_CONTINUE && r->status < NXT_HTTP_OK) { return 1; } - return nxt_http_request_is_bodyless_final(r, status); + return nxt_http_request_is_bodyless_final(r, r->status); } diff --git a/test/fake_upstream/README.md b/test/fake_upstream/README.md index db3ab7c50..b510d64a7 100644 --- a/test/fake_upstream/README.md +++ b/test/fake_upstream/README.md @@ -76,7 +76,7 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: | Port | Token | Mode (CLI) | Rust handler | pytest | |------|-------|-----------|--------------|--------| -| 7978 | — | — (plain-Python upstream in the test file) | `test_proxy_head.py` (bodyless upstream response: header block only, then close — not a `fake_upstream` mode, reserved here so the 79xx block stays in one registry) | +| 7978 | — | — (plain-Python upstream in the test file) | — | `test_proxy_head.py` (bodyless upstream response: header block, with or without trailing bytes, then close — not a `fake_upstream` mode, reserved here so the 79xx block stays in one registry) | | 7979 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_keepalive_disabled` (duplicate upstream Content-Length disables downstream keepalive for a keep-alive client; complete body keeps its terminal chunk) — out of the 7983–7999 block below, which is exhausted; 7980–7982 are `fake_otlp`, so 7974–7979 is the free gap | | 7983 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_raw` (zero raw `Content-Length` occurrences on the wire, complete re-framed chunked body) | | 7984 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl` (duplicate upstream Content-Length → neither forwarded, body re-framed, #113) | diff --git a/test/test_proxy_head.py b/test/test_proxy_head.py index 50db5f2c8..9be5b4e81 100644 --- a/test/test_proxy_head.py +++ b/test/test_proxy_head.py @@ -16,12 +16,18 @@ in the peer reader continues the exchange past it (that is pre-existing, see nxt_h1p_peer_header_parse), so it is left on the path it already had. -These cases also cover the two exits an unrelayed upstream header buffer can -take, but they cannot assert that it is released: a stranded request pool is -invisible from the client side, and a stock build exposes no pool counter -- -nxt_debug is never assigned, so even a --debug build emits no "mp ... release" -lines. That was measured instead with a throwaway build that forces nxt_debug -on and counts pools reaching a zero retain per request; see the commit message. +These cases also drive both shapes the header read can take -- with and +without body bytes alongside the header -- but they cannot assert that the +unrelayed header buffer is released: a stranded request pool is invisible from +the client side, and a stock build exposes no pool counter -- nxt_debug is +never assigned, so even a --debug build emits no "mp ... release" lines. That +was measured instead with a throwaway build that forces nxt_debug on and counts +pools reaching a zero retain per request; see the commit message. + +The same hold covers a third path nothing here reaches: an ordinary proxied +response whose body arrives in a read after the header, which stranded the pool +on master as well. It is unobservable from the client for the same reason, and +is covered by that measurement rather than by an assertion. See freeunitorg/freeunit#283 for the downstream half of the same rule. """ @@ -52,10 +58,10 @@ '/304': 'HTTP/1.1 304 Not Modified\r\nContent-Length: 10\r\n\r\n', # Header block and 10 junk bytes in a single write, so the bytes land in # the same read as the header. A response to HEAD has no body, so they - # must be dropped rather than relayed -- and the buffer holding them takes - # a different exit from nxt_h1p_peer_header_read_done() than the cases - # above, which is where an earlier revision of this change stranded the - # request pool. + # must be dropped rather than relayed -- and the buffer holding them + # reaches the bodyless branch of nxt_h1p_peer_header_read_done() with + # bytes still in it, unlike the cases above, which is where an earlier + # revision of this change stranded the request pool. '/cl10junk': ( 'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nJUNKJUNKJU' ), From 0c0f98114950f5fdc8d9de4f6c65c2037e001717 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 17:21:29 +0200 Subject: [PATCH 099/391] port: defer the last reference drop with an embedded work item nxt_port_use() (src/nxt_port.c:1163) funnels a port's last reference drop to port->engine when the drop happens on another thread, and it did that through nxt_port_post(), whose return value it ignored. nxt_port_post() allocates an nxt_port_work_t with nxt_zalloc() (src/nxt_port.c:1133) and returns NXT_ERROR when that fails. By then nxt_atomic_fetch_add() had already taken use_count to zero, so on failure nothing was queued and nxt_port_release() was never reached: the nxt_port_t, its memory pool, and the process reference the port holds (nxt_process_use(task, port->process, -1), src/nxt_port.c:206) all leaked, which also kept the whole nxt_process_t from ever reaching zero. That happens under exactly the memory pressure that caused it, so each occurrence makes the next more likely. There is nowhere to report the error from that call site -- the last reference is already given up, and releasing inline would free another engine's memory from this thread, which is what the deferral exists to prevent. So the failure path is removed rather than handled: the work item is embedded in nxt_port_t and posted straight to the engine with nxt_event_engine_post(), the shape nxt_runtime_process_release() uses for the process free (src/nxt_runtime.c:1657). A deferral that cannot allocate cannot fail. What the deferral carries is unchanged. The reference is handed to the item rather than given up: the count is left standing at 1 and the posted handler drops it with nxt_port_use(), so the release happens only if nobody else took a reference while the item was in flight. That re-check is not incidental -- the port stays reachable through process->ports until the item runs, and another engine can still take a reference in that window: nxt_process_broadcast_shm_ack() walks a process's ports through bare pointers (src/nxt_port_memory.c:1045) and nxt_port_socket_write() takes a reference on each (src/nxt_port_socket.c:351). Routing through nxt_port_post() provided the re-check, because that function took a reference of its own and nxt_port_post_handler() ended in nxt_port_use(port, -1); a handler that released unconditionally would free a port somebody holds. A single embedded item cannot be posted twice, so the foreign drop is a compare-and-set loop rather than a fetch-and-add followed by a compensating increment. The two-atomic form published a use_count of zero on a port that is still linked in process->ports and rt->ports, and a second thread promoting a bare pointer in that window can complete a 0 -> 1 -> 0 cycle of its own, reach the same branch, and post the same item again. nxt_locked_work_queue_add() (src/nxt_work_queue.c:230) links a work onto the queue tail, so a second post of an item that is already queued makes tail->next point at the item itself, and the engine draining that queue then walks a one-element cycle forever. nxt_port_post() could not produce that, because it allocated a separate item for every post. The compare-and-set never stores zero: a drop that is not the last one sets the count to c + i, and the last one stores 1 -- exactly one reference, whatever the size of the batch the drop carries, because the item's handler drops exactly one (nxt_port_socket_write() and nxt_port_error_handler() batch several into use_delta, src/nxt_port_socket.c:601 and :1500). use_count therefore reaches zero only on port->engine, immediately before nxt_port_release(). While the item is in flight the count is at least 1, so no other thread can observe a last drop, and at most one post is ever outstanding. The item becomes free again only once the handler's drop can run, which is after nxt_locked_work_queue_move() (src/nxt_work_queue.c:289) has taken it off the locked queue and copied it into the engine's own work queue -- the same copy that makes it safe for the release to free the pool the item lives in. nxt_port_release_handler(), the no-op that existed only to satisfy nxt_port_post()'s handler signature, goes away with the call. Three subtests in src/test/nxt_port_fail_test.c cover this. The first drops a port's last reference from a thread whose engine is not the port's and asserts that the item landing on the target engine's locked work queue is the port's own release_work -- false for any implementation that allocates it, whether or not the allocation succeeds -- that the item holds a reference, and that draining the port's engine reaches nxt_port_release(), observed through a cleanup registered on the port's memory pool. The second takes a reference between the post and the drain and asserts that the drain does not release, and that the release happens exactly once, when that other holder lets go. The third runs a racer thread on a third engine that watches use_count across a cross-engine last drop: the racer must never observe zero, the embedded item must be queued exactly once and not linked to itself -- the queue is walked with a step limit, so a self-link is an alert rather than a hang -- and the racer's own 1 -> 2 -> 1 with the item in flight must not post anything. Restoring the two-atomic form fails that subtest on every run. A fourth gives up two references in one cross-engine drop and asserts that the count left standing is 1, not 2, and that draining releases the port -- a hand-over that left the whole batch behind would strand the port through the other door. Fixes #187 Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Fable 5.1 --- src/nxt_port.c | 108 +++++- src/nxt_port.h | 9 + src/test/nxt_port_fail_test.c | 666 ++++++++++++++++++++++++++++++++++ 3 files changed, 774 insertions(+), 9 deletions(-) diff --git a/src/nxt_port.c b/src/nxt_port.c index 8dd9f1a61..eaea95b22 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -1153,27 +1153,117 @@ nxt_port_post(nxt_task_t *task, nxt_port_t *port, static void -nxt_port_release_handler(nxt_task_t *task, nxt_port_t *port, void *data) +nxt_port_release_work_handler(nxt_task_t *task, void *obj, void *data) { - /* no op */ + /* + * Drop the reference nxt_port_use() handed to this item, rather than + * releasing outright: this runs on port->engine, so the drop takes the + * on-engine branch of nxt_port_use() and releases only if nobody else + * took a reference while the item was in flight. That re-check is the + * whole point of holding a reference across the post -- see + * nxt_port_use(). + */ + + nxt_port_use(task, obj, -1); } void nxt_port_use(nxt_task_t *task, nxt_port_t *port, int i) { - int c; - - c = nxt_atomic_fetch_add(&port->use_count, i); + nxt_atomic_int_t c; - if (i < 0 && c == -i) { + if (i >= 0 + || port->engine == NULL + || task->thread->engine == port->engine) + { + c = nxt_atomic_fetch_add(&port->use_count, i); - if (port->engine == NULL || task->thread->engine == port->engine) { + if (i < 0 && c == -i) { nxt_port_release(task, port); + } - return; + return; + } + + /* + * A drop on another thread cannot release the port here -- the release + * frees port->mem_pool, and may drop the port's process reference, + * neither of which this thread owns -- so the last one is carried to + * port->engine by a work item. + * + * The item is embedded in the port and posted straight to the engine + * rather than routed through nxt_port_post(), which allocates one with + * nxt_zalloc() and can return NXT_ERROR. This call site has nowhere to + * put that error: the reference it is dropping is the last, so refusing + * to defer would leak the port, its memory pool and the process + * reference it holds -- unbounded, under exactly the memory pressure + * that caused the failure -- while releasing here would free another + * engine's memory from this thread, which is what the deferral exists + * to prevent. A deferral that cannot allocate cannot fail. This + * mirrors nxt_runtime_process_release(). + * + * The reference is handed to the item instead of being given up: the + * count is left standing at 1 and the posted handler drops it on + * port->engine. The port stays reachable through process->ports until + * then, so another engine can still take a reference in that window -- + * nxt_process_broadcast_shm_ack() walks a process's ports through bare + * pointers and nxt_port_socket_write() takes a reference on each -- and + * only a drop re-checked at the far end can tell that apart from a port + * nobody wants. Releasing from the handler unconditionally would free + * a port somebody holds. Routing through nxt_port_post() used to + * provide exactly this: it took a reference of its own and ended in + * nxt_port_use(port, -1). + * + * The hand-over is why the loop below is a compare-and-set that stores + * the item's one reference in place of the last drop, rather than a + * fetch-and-add to zero followed by a compensating increment. use_count + * reaches zero only on port->engine, immediately before + * nxt_port_release(), so while the item is in flight no other thread can + * observe a last drop: a reference taken in that window takes the count + * to 2, and the holder's drop finds the item's own reference still + * standing, so it is not the last and cannot reach this branch. At most + * one post is therefore ever in flight, and the item cannot be posted + * twice. That matters because nxt_locked_work_queue_add() links the + * item onto the queue tail: an item posted while it is already queued + * becomes its own successor, and the engine draining the queue then + * spins forever. + * + * The item becomes free again only once the handler's drop can run, and + * that is after nxt_locked_work_queue_move() has taken it off the + * locked queue and copied it into the engine's own work queue -- which + * is also why the release may free the pool the item lives in. + */ + + for ( ;; ) { + c = port->use_count; + + if (c != -i) { + nxt_assert(c > -i); + + if (nxt_atomic_cmp_set(&port->use_count, c, c + i)) { + return; + } + + continue; } - nxt_port_post(task, port, nxt_port_release_handler, NULL); + /* + * Exactly one reference is left standing, whatever the size of the + * batch this drop carries, because the item's handler drops exactly + * one. + */ + + if (nxt_atomic_cmp_set(&port->use_count, c, 1)) { + break; + } } + + port->release_work.handler = nxt_port_release_work_handler; + port->release_work.task = &port->engine->task; + port->release_work.obj = port; + port->release_work.data = NULL; + port->release_work.next = NULL; + + nxt_event_engine_post(port->engine, &port->release_work); } diff --git a/src/nxt_port.h b/src/nxt_port.h index e1aa65359..d227129a7 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -300,6 +300,15 @@ struct nxt_port_s { nxt_mp_t *mem_pool; nxt_event_engine_t *engine; + /* + * The deferral that carries the last reference drop to port->engine. + * Embedded rather than allocated, so that nxt_port_use() has no failure + * path -- see the comment there. Single-instance: use_count reaches + * zero only on port->engine, so only one thread at a time can hand the + * last reference over, and at most one post is ever in flight. + */ + nxt_work_t release_work; + nxt_buf_t *free_bufs; nxt_socket_t pair[2]; diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index 1e1722ba3..ddd070468 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -18,6 +18,17 @@ static nxt_int_t nxt_port_fail_test_socket_write(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_error_handler(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_mp_baseline(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_release(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_acquire(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_race(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_batch(nxt_thread_t *thr); +static void nxt_port_fail_test_racer(void *data); +static nxt_int_t nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, + nxt_work_t *item); +static void nxt_port_fail_test_engine_signal(nxt_event_engine_t *engine, + nxt_uint_t signo); +static void nxt_port_fail_test_released(nxt_task_t *task, void *obj, + void *data); static nxt_int_t nxt_port_fail_test_sender_pattern(nxt_task_t *task, nxt_port_t *port, nxt_mp_t *mp); static void nxt_port_fail_test_mp_completion(nxt_task_t *task, void *obj, @@ -30,6 +41,8 @@ static nxt_int_t nxt_port_fail_test_fd_count(void); static nxt_uint_t nxt_port_fail_test_completions; +static nxt_uint_t nxt_port_fail_test_releases; +static nxt_uint_t nxt_port_fail_test_signals; nxt_int_t @@ -54,6 +67,22 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_cross_engine_release(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_cross_engine_acquire(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_cross_engine_race(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_fail_test_cross_engine_batch(thr) != NXT_OK) { + return NXT_ERROR; + } + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "port failure test passed"); @@ -580,6 +609,643 @@ nxt_port_fail_test_sender_pattern(nxt_task_t *task, nxt_port_t *port, } +/* + * A last reference dropped from a thread other than port->engine's must be + * deferred to that engine, and the deferral must not be able to fail. + * + * nxt_port_use() used to route the drop through nxt_port_post(), which + * nxt_zalloc()s an nxt_port_work_t and returns NXT_ERROR when that fails. + * The return value was ignored: use_count was already zero, nothing was + * queued, and the port, its memory pool and the process reference it holds + * leaked (issue #187). + * + * There is no allocation-failure injection hook reachable from here, so the + * regression is pinned structurally instead: the item that lands on the + * foreign engine's locked work queue must be the one embedded in the port. + * That is false for any implementation that allocates it, whether or not the + * allocation succeeds. Draining then has to reach nxt_port_release(), which + * is observed through a cleanup on the port's own memory pool. + */ + +static nxt_int_t +nxt_port_fail_test_cross_engine_release(nxt_thread_t *thr) +{ + nxt_task_t *task; + nxt_port_t *port; + nxt_work_t *posted; + nxt_event_engine_t current, foreign; + + task = thr->task; + task->thread = thr; + + /* + * Two minimal engines: the one this thread runs on, and the port's. + * They only have to differ and to carry a work queue; the post path + * touches locked_work_queue, event.signal and task. + */ + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + + /* + * Without this stub nxt_event_engine_signal() would fall through to + * writing on foreign.pipe, which a bare engine does not have. + */ + foreign.event.signal = nxt_port_fail_test_engine_signal; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_engine; + } + + port->engine = &foreign; + + if (nxt_slow_path(nxt_mp_cleanup(port->mem_pool, + nxt_port_fail_test_released, + task, port, NULL) != NXT_OK)) + { + nxt_port_use(task, port, -1); + goto fail_engine; + } + + nxt_port_fail_test_releases = 0; + nxt_port_fail_test_signals = 0; + + posted = &port->release_work; + + nxt_port_use(task, port, -1); + + /* The release must be deferred, not run on this thread. */ + if (nxt_slow_path(nxt_port_fail_test_releases != 0)) { + nxt_log_alert(thr->log, "port fail test: cross-engine release ran " + "on the calling thread"); + goto fail_engine; + } + + if (nxt_slow_path(nxt_port_fail_test_signals != 1)) { + nxt_log_alert(thr->log, "port fail test: cross-engine release did " + "not signal the target engine (%ui)", + nxt_port_fail_test_signals); + goto fail_engine; + } + + /* + * The heart of the regression check: an allocated work item would make + * this a different pointer, and a failed allocation would leave the + * queue empty. + */ + if (nxt_slow_path(foreign.locked_work_queue.head != posted)) { + nxt_log_alert(thr->log, "port fail test: the posted item is not the " + "port's embedded release work"); + goto fail_engine; + } + + /* + * The reference is handed to the item, not given up: the count is back + * at 1 and the posted handler is what drops it. + */ + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: the deferred release does " + "not hold a reference (use_count %A)", port->use_count); + goto fail_engine; + } + + /* The handler runs on the port's engine, so this thread becomes it. */ + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_releases != 1)) { + nxt_log_alert(thr->log, "port fail test: draining the target engine " + "did not release the port (%ui)", + nxt_port_fail_test_releases); + goto fail_engine; + } + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_OK; + +fail_engine: + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +/* + * A cross-engine last drop leaves the port reachable through + * process->ports until the deferred item runs, so another engine can still + * take a reference in that window -- nxt_process_broadcast_shm_ack() does. + * The deferral therefore has to hand its reference over and re-check at the + * far end, which is what routing through nxt_port_post() used to provide: + * it took a reference of its own and ended in nxt_port_use(port, -1). + * + * A deferred handler that released unconditionally would free a port + * somebody holds -- a use-after-free, and only an assertion in a --debug + * build. Here the acquisition is made between the post and the drain: the + * drain must not release, and the release must happen exactly once, when + * the other holder drops its reference. + */ + +static nxt_int_t +nxt_port_fail_test_cross_engine_acquire(nxt_thread_t *thr) +{ + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t current, foreign; + + task = thr->task; + task->thread = thr; + + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + foreign.event.signal = nxt_port_fail_test_engine_signal; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_engine; + } + + port->engine = &foreign; + + if (nxt_slow_path(nxt_mp_cleanup(port->mem_pool, + nxt_port_fail_test_released, + task, port, NULL) != NXT_OK)) + { + nxt_port_use(task, port, -1); + goto fail_engine; + } + + nxt_port_fail_test_releases = 0; + + nxt_port_use(task, port, -1); + + /* + * The concurrent acquisition, made while the item is queued. A real one + * comes from another engine; what matters here is that it happens after + * the drop that posted the item and before that item runs. + */ + nxt_port_use(task, port, 1); + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_releases != 0)) { + nxt_log_alert(thr->log, "port fail test: the deferred release freed " + "a port another reference was taken on"); + goto fail_port; + } + + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: use_count is %A after the " + "deferred release, expected 1", port->use_count); + goto fail_port; + } + + /* Now the other holder lets go, on the port's own engine. */ + nxt_port_use(task, port, -1); + + if (nxt_slow_path(nxt_port_fail_test_releases != 1)) { + nxt_log_alert(thr->log, "port fail test: dropping the last reference " + "released the port %ui times, expected 1", + nxt_port_fail_test_releases); + goto fail_engine; + } + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_OK; + +fail_port: + + nxt_port_use(task, port, -1); + +fail_engine: + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +/* + * The hand-over must not be able to post the embedded item twice. + * + * The item is a single object, so two threads that both believe they are + * dropping the last reference would post it twice, and + * nxt_locked_work_queue_add() links a work onto the queue tail: the second + * post makes tail->next point at the item itself, and the engine draining + * that queue then walks a one-element cycle forever. Routing through + * nxt_port_post() could not produce this, because it allocated a separate + * item for every post. + * + * The shape that would produce it is a foreign drop that publishes a zero + * use_count: another engine promoting a bare port pointer -- + * nxt_process_broadcast_shm_ack() walks a process's ports that way, and + * nxt_port_socket_write() takes a reference on each -- can then complete a + * 0 -> 1 -> 0 cycle of its own and reach the deferral as well. So the + * invariant this pins is the one that makes it impossible: use_count is + * never observable as zero off port->engine. + * + * A racer thread on a third engine watches use_count across a cross-engine + * last drop and reports every zero it sees; then, with the item already in + * flight, it takes a reference and drops it again -- the 1 -> 2 -> 1 that + * must not post anything. The locked queue is walked with a step limit, so + * a self-linked item fails the test instead of hanging it. + */ + +#define NXT_PORT_FAIL_TEST_RACE_TRIALS 128 +#define NXT_PORT_FAIL_TEST_RACE_SPINS 4000000 +#define NXT_PORT_FAIL_TEST_QUEUE_LIMIT 16 + + +typedef struct { + nxt_port_t *port; + nxt_event_engine_t *engine; + nxt_atomic_t ready; + nxt_atomic_t stop; + nxt_uint_t zeroes; +} nxt_port_fail_test_race_t; + + +static nxt_int_t +nxt_port_fail_test_cross_engine_race(nxt_thread_t *thr) +{ + nxt_int_t queued; + nxt_task_t *task; + nxt_uint_t trial; + nxt_port_t *port; + nxt_thread_link_t *link; + nxt_thread_handle_t handle; + nxt_event_engine_t current, foreign, other; + nxt_port_fail_test_race_t race; + + task = thr->task; + task->thread = thr; + + nxt_memzero(&other, sizeof(other)); + + for (trial = 0; trial < NXT_PORT_FAIL_TEST_RACE_TRIALS; trial++) { + + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + foreign.event.signal = nxt_port_fail_test_engine_signal; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_engine; + } + + port->engine = &foreign; + + if (nxt_slow_path(nxt_mp_cleanup(port->mem_pool, + nxt_port_fail_test_released, + task, port, NULL) != NXT_OK)) + { + nxt_port_use(task, port, -1); + goto fail_engine; + } + + nxt_port_fail_test_releases = 0; + nxt_port_fail_test_signals = 0; + + nxt_memzero(&race, sizeof(race)); + race.port = port; + race.engine = &other; + + link = nxt_zalloc(sizeof(nxt_thread_link_t)); + if (nxt_slow_path(link == NULL)) { + goto fail_port; + } + + link->start = nxt_port_fail_test_racer; + link->work.data = ∽̱ + + if (nxt_slow_path(nxt_thread_create(&handle, link) != NXT_OK)) { + goto fail_port; + } + + /* + * The racer has to be spinning before the drop, or the window it + * watches for is gone before it looks. + */ + + while (race.ready == 0) { + nxt_cpu_pause(); + } + + nxt_port_use(task, port, -1); + + race.stop = 1; + + nxt_thread_wait(handle); + + if (nxt_slow_path(race.zeroes != 0)) { + nxt_log_alert(thr->log, "port fail test: a foreign thread saw " + "use_count 0 on a live port %ui times", race.zeroes); + goto fail_port; + } + + queued = nxt_port_fail_test_queued(&foreign.locked_work_queue, + &port->release_work); + + if (nxt_slow_path(queued < 0)) { + nxt_log_alert(thr->log, "port fail test: the embedded release " + "work is linked into the target engine's queue " + "more than once"); + goto fail_engine; + } + + if (nxt_slow_path(queued != 1)) { + nxt_log_alert(thr->log, "port fail test: the embedded release " + "work is queued %i times, expected 1", queued); + goto fail_port; + } + + if (nxt_slow_path(nxt_port_fail_test_signals != 1)) { + nxt_log_alert(thr->log, "port fail test: the target engine was " + "signalled %ui times, expected 1", + nxt_port_fail_test_signals); + goto fail_port; + } + + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: use_count is %A with " + "the deferral in flight, expected 1", + port->use_count); + goto fail_port; + } + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_releases != 1)) { + nxt_log_alert(thr->log, "port fail test: draining the target " + "engine released the port %ui times, expected 1", + nxt_port_fail_test_releases); + goto fail_engine; + } + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + } + + thr->engine = NULL; + + return NXT_OK; + +fail_port: + + nxt_port_use(task, port, -1); + +fail_engine: + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +static void +nxt_port_fail_test_racer(void *data) +{ + nxt_task_t task; + nxt_uint_t i; + nxt_thread_t *thr; + nxt_port_fail_test_race_t *race; + + race = data; + + thr = nxt_thread(); + thr->engine = race->engine; + + nxt_memzero(&task, sizeof(task)); + task.thread = thr; + task.log = thr->log; + + race->ready = 1; + + for (i = 0; i < NXT_PORT_FAIL_TEST_RACE_SPINS; i++) { + + if (nxt_slow_path(race->port->use_count == 0)) { + race->zeroes++; + break; + } + + if (race->stop != 0) { + break; + } + + nxt_cpu_pause(); + } + + /* + * The acquisition another engine can still make while the deferral is + * in flight, and the drop that follows it: 1 -> 2 -> 1, which must not + * reach the deferral and must not post anything. + */ + + nxt_port_use(&task, race->port, 1); + nxt_port_use(&task, race->port, -1); +} + + +/* + * Count the occurrences of one work item in a locked work queue, refusing to + * walk further than the queue can legitimately be. A work item posted twice + * becomes its own successor, so the walk would otherwise not terminate -- + * which is exactly what the draining engine does. + */ + +static nxt_int_t +nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, nxt_work_t *item) +{ + nxt_int_t n, steps; + nxt_work_t *w; + + n = 0; + + for (w = lwq->head, steps = 0; w != NULL; w = w->next, steps++) { + + if (steps >= NXT_PORT_FAIL_TEST_QUEUE_LIMIT) { + return -1; + } + + if (w == item) { + n++; + } + } + + return n; +} + + +/* + * A drop can carry more than one reference at once: nxt_port_socket_write() + * and nxt_port_error_handler() batch theirs into use_delta + * (src/nxt_port_socket.c:601, src/nxt_port_socket.c:1500), so a cross-engine + * drop of -2 or more can be the last one. + * + * The deferral must then leave exactly one reference standing, whatever the + * size of the batch, because the posted handler drops exactly one. A + * hand-over that left the whole batch behind would strand the port, its + * memory pool and the process reference it holds -- the very leak this + * change exists to remove, reintroduced through the other door. + */ + +static nxt_int_t +nxt_port_fail_test_cross_engine_batch(nxt_thread_t *thr) +{ + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t current, foreign; + + task = thr->task; + task->thread = thr; + + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + foreign.event.signal = nxt_port_fail_test_engine_signal; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail_engine; + } + + port->engine = &foreign; + + if (nxt_slow_path(nxt_mp_cleanup(port->mem_pool, + nxt_port_fail_test_released, + task, port, NULL) != NXT_OK)) + { + nxt_port_use(task, port, -1); + goto fail_engine; + } + + nxt_port_fail_test_releases = 0; + + /* + * Two references, given up by one drop, from a thread that is not the + * port's engine. + */ + + nxt_port_use(task, port, 1); + + nxt_port_use(task, port, -2); + + if (nxt_slow_path(nxt_port_fail_test_releases != 0)) { + nxt_log_alert(thr->log, "port fail test: a batched cross-engine " + "release ran on the calling thread"); + goto fail_engine; + } + + if (nxt_slow_path(foreign.locked_work_queue.head != &port->release_work)) { + nxt_log_alert(thr->log, "port fail test: a batched cross-engine last " + "drop did not post the port's embedded release work"); + goto fail_engine; + } + + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: use_count is %A after a " + "batched cross-engine last drop, expected 1", + port->use_count); + goto fail_port; + } + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_releases != 1)) { + nxt_log_alert(thr->log, "port fail test: draining the target engine " + "released the batched port %ui times, expected 1", + nxt_port_fail_test_releases); + goto fail_engine; + } + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_OK; + +fail_port: + + nxt_port_use(task, port, -1); + +fail_engine: + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + thr->engine = NULL; + + return NXT_ERROR; +} + + +static void +nxt_port_fail_test_engine_signal(nxt_event_engine_t *engine, nxt_uint_t signo) +{ + nxt_port_fail_test_signals++; +} + + +static void +nxt_port_fail_test_released(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_fail_test_releases++; +} + + static void nxt_port_fail_test_mp_completion(nxt_task_t *task, void *obj, void *data) { From fe6df7153340da6686ddaaeb105941aa54c6bb67 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 17:50:15 +0200 Subject: [PATCH 100/391] port: take the app port under a lock in the shared memory ack nxt_process_broadcast_shm_ack() (src/nxt_port_memory.c:1000) holds a reference to the process, but reaches the app port through a lookup: nxt_process_port_first() on process->ports, with no lock. It hands that bare pointer to nxt_port_post(), whose unconditional nxt_atomic_fetch_add(&port->use_count, 1) (src/nxt_port.c:1139) can land on a port that has already dropped its last reference. That happens whenever an OOSM-blocked application is killed under load. On a router worker engine, nxt_port_mmap_buf_completion() (src/nxt_port_memory.c:230) wins cmp_set(&hdr->oosm, 1, 0) as a stalled client's response buffer finishes, and broadcasts the ack. On the router's main thread, SIGCHLD -> nxt_port_remove_pid() (src/nxt_port.c:1055) -> nxt_process_close_ports() has taken the app port's references away, and nxt_port_release() (src/nxt_port.c:194) is unlinking it and destroying its memory pool. The +1 arrives a few hundred nanoseconds before nxt_port_mp_cleanup() (src/nxt_port.c:101) asserts port->use_count == 0 -- and in a release build, where that assertion is compiled out, it is a use-after-free of the port and of the work item posted with it. A reference derived from a lookup cannot be taken with nxt_port_use(). Add nxt_port_use_unless_zero(), a CAS loop that refuses a port whose count has already reached zero, and pair it with the lock that decides whether a port is still reachable: nxt_port_release() now unlinks the port under rt->processes_mutex, and the broadcast reads process->ports and try-refs the port under the same mutex. {find, ref} and {last drop, unlink} are then mutually exclusive, so the broadcast either gets a port that stays alive for the length of the post, or gets nothing -- which is the right answer, since the process is going away and its application will not wait for shared memory again. rt->processes_mutex is a documented leaf (src/nxt_process.c:150), and stays one: only the lookup is under it. nxt_process_use() takes it itself, so both the process reference in the broadcast and the one dropped by nxt_port_release() are handled outside the critical section. src/test/nxt_port_use_unless_zero_test.c pins the primitive: a live port hands out references, a port whose last reference is gone refuses them and keeps its count at zero, and neither holds only under contention. The unconditional increment that this replaces fails it on the first attempt. test_process_shm_oosm_kill drives the router path itself -- three clients stalled on a one-segment application, its worker SIGKILLed five times -- which reproduces the assertion on an unfixed --debug build, and is a use-after-free driver for the sanitizer legs. Fixes #195 Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + src/nxt_port.c | 56 ++++ src/nxt_port.h | 1 + src/nxt_port_memory.c | 88 ++++-- src/test/nxt_port_use_unless_zero_test.c | 378 +++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/test_process_shm_oosm.py | 188 +++++++++++ 8 files changed, 697 insertions(+), 20 deletions(-) create mode 100644 src/test/nxt_port_use_unless_zero_test.c diff --git a/auto/sources b/auto/sources index 0f837b29d..0526baebc 100644 --- a/auto/sources +++ b/auto/sources @@ -176,6 +176,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_http_chunk_parse_test.c \ src/test/nxt_conf_parse_test.c \ src/test/nxt_port_fail_test.c \ + src/test/nxt_port_use_unless_zero_test.c \ src/test/nxt_port_mmap_range_test.c \ src/test/nxt_port_ready_test.c \ src/test/nxt_router_new_port_test.c \ diff --git a/src/nxt_port.c b/src/nxt_port.c index eaea95b22..e5f184c23 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -190,9 +190,46 @@ nxt_port_close(nxt_task_t *task, nxt_port_t *port) } +/* + * Take a reference, unless the port has already dropped its last one. + * + * This is the counterpart of a lookup in process->ports: a reference derived + * from a pointer that was found rather than from one the caller already + * holds cannot use nxt_port_use(), because the port may be in teardown by + * the time the increment lands. Callers must hold rt->processes_mutex, + * which nxt_port_release() also holds while it unlinks the port, so that + * {find, ref} and {last drop, unlink} are mutually exclusive: a port still + * reachable through process->ports either still has a reference, or has + * already had its count latched at zero and fails here. + * + * Returns 1 when a reference was taken -- the caller then owns it and must + * drop it with nxt_port_use(task, port, -1), outside the mutex. + */ + +nxt_bool_t +nxt_port_use_unless_zero(nxt_port_t *port) +{ + nxt_atomic_int_t c; + + for ( ;; ) { + c = port->use_count; + + if (c <= 0) { + return 0; + } + + if (nxt_atomic_cmp_set(&port->use_count, c, c + 1)) { + return 1; + } + } +} + + static void nxt_port_release(nxt_task_t *task, nxt_port_t *port) { + nxt_runtime_t *rt; + nxt_debug(task, "port %p %d:%d release, type %d", port, port->pid, port->id, port->type); @@ -201,8 +238,27 @@ nxt_port_release(nxt_task_t *task, nxt_port_t *port) if (port->link.next != NULL) { nxt_assert(port->process != NULL); + rt = task->thread->runtime; + + /* + * The unlink happens under rt->processes_mutex so that it cannot + * race a lookup in process->ports on another engine -- see + * nxt_port_use_unless_zero(). use_count is zero here and stays + * zero, so every reader that reaches the port through the queue + * before the unlink takes the mutex and fails the try-ref, and every + * reader after it cannot reach the port at all. + * + * rt->processes_mutex must stay a leaf (src/nxt_process.c:150), so + * the process reference is dropped after the unlock: + * nxt_process_use() takes the same mutex itself. + */ + + nxt_thread_mutex_lock(&rt->processes_mutex); + nxt_process_port_remove(port); + nxt_thread_mutex_unlock(&rt->processes_mutex); + nxt_process_use(task, port->process, -1); } diff --git a/src/nxt_port.h b/src/nxt_port.h index d227129a7..83defcfab 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -439,6 +439,7 @@ void nxt_port_empty_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); nxt_int_t nxt_port_post(nxt_task_t *task, nxt_port_t *port, nxt_port_post_handler_t handler, void *data); void nxt_port_use(nxt_task_t *task, nxt_port_t *port, int i); +nxt_bool_t nxt_port_use_unless_zero(nxt_port_t *port); nxt_inline void nxt_port_inc_use(nxt_port_t *port) { diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 02dcb480f..cead5e39b 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -1003,35 +1003,83 @@ nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) void nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process) { - nxt_port_t *port; + nxt_port_t *port; + nxt_runtime_t *rt; - if (nxt_slow_path(process == NULL || nxt_queue_is_empty(&process->ports))) - { + if (nxt_slow_path(process == NULL)) { return; } - port = nxt_process_port_first(process); + rt = task->thread->runtime; - if (port->type == NXT_PROCESS_APP) { - /* - * The process is handed to another engine as a bare pointer and is - * dereferenced there, so the posted handler needs a reference of its - * own and drops it when it is done. nxt_port_post() returns NXT_OK - * both when it queues the handler and when it calls it inline on the - * current engine, and the handler owns the reference either way; only - * the NXT_ERROR case leaves nothing to run, so only that case has to - * be undone here. The inline call is safe: the caller still holds - * its own reference, so this one cannot be the last. - */ + /* + * This runs on any engine -- most often a worker engine, from + * nxt_port_mmap_buf_completion() -- while the app port's teardown runs on + * the router's main thread. The caller holds a reference to the process, + * not to the port, so the port has to be looked up, and a reference taken + * from a lookup can land on a port whose count already reached zero: + * nxt_port_post() below does an unconditional increment, and + * nxt_port_release() would by then be destroying the port's memory pool + * (issue #195 -- the `port->use_count == 0` assertion in a debug build, + * a use-after-free of the port and of the work item in a release one). + * + * rt->processes_mutex is the lock nxt_port_release() unlinks the port + * under, so reading process->ports and try-referencing the port under it + * cannot observe a port that is being released: either the try-ref wins + * and the port is alive for as long as this function holds it, or the + * port is already dying and there is nothing to acknowledge -- the + * process is going away and the app will not wait for shared memory + * again. + * + * Only the lookup is done under the mutex. Everything after it takes + * other locks (nxt_process_use() takes this very mutex), and + * rt->processes_mutex is a leaf -- see src/nxt_process.c:150. + */ - nxt_process_use(task, process, 1); + nxt_thread_mutex_lock(&rt->processes_mutex); - if (nxt_slow_path(nxt_port_post(task, port, nxt_port_broadcast_shm_ack, - process) != NXT_OK)) - { - nxt_process_use(task, process, -1); + port = NULL; + + if (!nxt_queue_is_empty(&process->ports)) { + port = nxt_process_port_first(process); + + if (port->type != NXT_PROCESS_APP || !nxt_port_use_unless_zero(port)) { + port = NULL; } } + + nxt_thread_mutex_unlock(&rt->processes_mutex); + + if (port == NULL) { + return; + } + + /* + * The process is handed to another engine as a bare pointer and is + * dereferenced there, so the posted handler needs a reference of its + * own and drops it when it is done. nxt_port_post() returns NXT_OK + * both when it queues the handler and when it calls it inline on the + * current engine, and the handler owns the reference either way; only + * the NXT_ERROR case leaves nothing to run, so only that case has to + * be undone here. The inline call is safe: the caller still holds + * its own reference, so this one cannot be the last. + */ + + nxt_process_use(task, process, 1); + + if (nxt_slow_path(nxt_port_post(task, port, nxt_port_broadcast_shm_ack, + process) != NXT_OK)) + { + nxt_process_use(task, process, -1); + } + + /* + * nxt_port_post() took its own reference on success, so the lookup + * reference is dropped here either way. It is dropped after the post so + * that the port cannot be released between the try-ref and the post. + */ + + nxt_port_use(task, port, -1); } diff --git a/src/test/nxt_port_use_unless_zero_test.c b/src/test/nxt_port_use_unless_zero_test.c new file mode 100644 index 000000000..72dc3f526 --- /dev/null +++ b/src/test/nxt_port_use_unless_zero_test.c @@ -0,0 +1,378 @@ +/* + * Copyright (C) NGINX, Inc. + */ + +#include +#include +#include "nxt_tests.h" + +#include + + +#define NXT_PORT_UUZ_RACERS 8 +#define NXT_PORT_UUZ_ROUNDS 2000 + + +typedef struct { + nxt_port_t *port; + nxt_atomic_t gen; + nxt_atomic_t done; + nxt_atomic_t released; + nxt_atomic_t wins; + nxt_atomic_t late_wins; + nxt_atomic_t stop; +} nxt_port_uuz_ctx_t; + + +static nxt_int_t nxt_port_uuz_test_semantics(nxt_thread_t *thr); +static nxt_int_t nxt_port_uuz_test_after_last_drop(nxt_thread_t *thr); +static nxt_int_t nxt_port_uuz_test_race(nxt_thread_t *thr); +static void *nxt_port_uuz_racer(void *data); +static nxt_port_t *nxt_port_uuz_port(nxt_task_t *task); +static void nxt_port_uuz_free(nxt_port_t *port); + + +nxt_int_t +nxt_port_use_unless_zero_test(nxt_thread_t *thr) +{ + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port use_unless_zero test started"); + + if (nxt_port_uuz_test_semantics(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_uuz_test_after_last_drop(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_uuz_test_race(thr) != NXT_OK) { + return NXT_ERROR; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port use_unless_zero test passed"); + + return NXT_OK; +} + + +/* A live port hands out references, and the count follows them exactly. */ + +static nxt_int_t +nxt_port_uuz_test_semantics(nxt_thread_t *thr) +{ + nxt_port_t *port; + + port = nxt_port_uuz_port(thr->task); + if (nxt_slow_path(port == NULL)) { + return NXT_ERROR; + } + + if (nxt_slow_path(!nxt_port_use_unless_zero(port))) { + nxt_log_alert(thr->log, "port use_unless_zero test: a live port " + "(use_count %A) refused a reference", port->use_count); + goto fail; + } + + if (nxt_slow_path(port->use_count != 2)) { + nxt_log_alert(thr->log, "port use_unless_zero test: use_count is %A, " + "expected 2", port->use_count); + goto fail; + } + + if (nxt_slow_path(!nxt_port_use_unless_zero(port) + || port->use_count != 3)) + { + nxt_log_alert(thr->log, "port use_unless_zero test: a second " + "reference did not stack (use_count %A)", + port->use_count); + goto fail; + } + + nxt_atomic_fetch_add(&port->use_count, -2); + + if (nxt_slow_path(port->use_count != 1)) { + nxt_log_alert(thr->log, "port use_unless_zero test: use_count is %A " + "after two drops, expected 1", port->use_count); + goto fail; + } + + nxt_port_uuz_free(port); + + return NXT_OK; + +fail: + + nxt_port_uuz_free(port); + + return NXT_ERROR; +} + + +/* + * The regression itself (issue #195). + * + * nxt_process_broadcast_shm_ack() reaches an app port through + * process->ports, which is a lookup: the port may already have dropped its + * last reference and be inside nxt_port_release(), destroying its memory + * pool. The unconditional nxt_atomic_fetch_add() that nxt_port_post() does + * on that pointer resurrects a dead port -- the `use_count == 0` assertion + * in nxt_port_mp_cleanup() in a debug build, a use-after-free in a release + * one. + * + * The state reproduced here is exactly the window: use_count has reached + * zero and the port is still linked, because nxt_port_release() unlinks it + * only afterwards. A try-ref must fail, and must leave the count alone -- + * an implementation that increments first and inspects afterwards leaves 1 + * here and fails the same way the router did. + */ + +static nxt_int_t +nxt_port_uuz_test_after_last_drop(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_uint_t i; + nxt_port_t *port; + + port = nxt_port_uuz_port(thr->task); + if (nxt_slow_path(port == NULL)) { + return NXT_ERROR; + } + + ret = NXT_OK; + + /* The last drop, without the release nxt_port_use() would run. */ + nxt_atomic_fetch_add(&port->use_count, -1); + + for (i = 0; i < 3; i++) { + if (nxt_slow_path(nxt_port_use_unless_zero(port))) { + nxt_log_alert(thr->log, "port use_unless_zero test: a port whose " + "last reference is gone handed out one anyway " + "(attempt %ui)", i + 1); + ret = NXT_ERROR; + break; + } + + if (nxt_slow_path(port->use_count != 0)) { + nxt_log_alert(thr->log, "port use_unless_zero test: a refused " + "try-ref changed use_count to %A", port->use_count); + ret = NXT_ERROR; + break; + } + } + + port->use_count = 0; + + nxt_port_uuz_free(port); + + return ret; +} + + +/* + * The same thing under contention, with the ordering the mutex in + * nxt_process_broadcast_shm_ack() and nxt_port_release() guarantees: the + * racers only look the port up after the owner has dropped the last + * reference. Every one of them must be refused. + * + * The rounds also exercise the CAS loop against concurrent increments, so a + * lost update would show up as a non-zero count at the end of a round. + */ + +static nxt_int_t +nxt_port_uuz_test_race(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_uint_t i, round; + pthread_t racers[NXT_PORT_UUZ_RACERS]; + nxt_port_uuz_ctx_t ctx; + + nxt_memzero(&ctx, sizeof(ctx)); + + ctx.port = nxt_port_uuz_port(thr->task); + if (nxt_slow_path(ctx.port == NULL)) { + return NXT_ERROR; + } + + /* + * The port outlives every round: the count is put back by hand rather + * than letting the port be released, so that the racers never touch + * freed memory even when the implementation under test is wrong. + */ + ctx.port->use_count = 0; + + for (i = 0; i < NXT_PORT_UUZ_RACERS; i++) { + if (nxt_slow_path(pthread_create(&racers[i], NULL, nxt_port_uuz_racer, + &ctx) != 0)) + { + nxt_log_alert(thr->log, "port use_unless_zero test: " + "pthread_create() failed"); + + ctx.stop = 1; + nxt_atomic_fetch_add(&ctx.gen, 1); + + while (i > 0) { + pthread_join(racers[--i], NULL); + } + + nxt_port_uuz_free(ctx.port); + + return NXT_ERROR; + } + } + + ret = NXT_OK; + + for (round = 0; round < NXT_PORT_UUZ_ROUNDS; round++) { + ctx.port->use_count = 1; + ctx.released = 0; + ctx.late_wins = 0; + ctx.done = 0; + + nxt_atomic_fetch_add(&ctx.gen, 1); + + /* Contended try-refs, which may or may not win. */ + nxt_thread_yield(); + + if (nxt_atomic_fetch_add(&ctx.port->use_count, -1) != 1) { + /* + * A racer that won still holds a reference; wait for it. It + * cannot be held for long -- every winner drops immediately. + */ + while (ctx.port->use_count != 0) { + nxt_thread_yield(); + } + } + + /* From here on no lookup may produce a reference. */ + ctx.released = 1; + + while (ctx.done != NXT_PORT_UUZ_RACERS) { + nxt_thread_yield(); + } + + if (nxt_slow_path(ctx.port->use_count != 0)) { + nxt_log_alert(thr->log, "port use_unless_zero test: use_count is " + "%A after round %ui, expected 0", + ctx.port->use_count, round); + ret = NXT_ERROR; + break; + } + + if (nxt_slow_path(ctx.late_wins != 0)) { + nxt_log_alert(thr->log, "port use_unless_zero test: %A " + "reference(s) taken after the last drop, in round " + "%ui", ctx.late_wins, round); + ret = NXT_ERROR; + break; + } + } + + ctx.stop = 1; + nxt_atomic_fetch_add(&ctx.gen, 1); + + for (i = 0; i < NXT_PORT_UUZ_RACERS; i++) { + pthread_join(racers[i], NULL); + } + + if (ret == NXT_OK && ctx.wins == 0) { + /* + * Not a failure of the code under test, but nothing was contended, + * so say so rather than pass quietly. + */ + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port use_unless_zero test: " + "no contended try-ref won; the race was not exercised"); + } + + ctx.port->use_count = 0; + + nxt_port_uuz_free(ctx.port); + + return ret; +} + + +static void * +nxt_port_uuz_racer(void *data) +{ + nxt_bool_t late; + nxt_atomic_int_t gen, last_gen; + nxt_port_uuz_ctx_t *ctx; + + ctx = data; + last_gen = 0; + + for ( ;; ) { + for ( ;; ) { + gen = ctx->gen; + + if (gen != last_gen) { + break; + } + + nxt_thread_yield(); + } + + last_gen = gen; + + if (ctx->stop) { + return NULL; + } + + /* + * Whether this lookup is racing the owner's last drop or is + * unambiguously after it decides what the outcome is allowed to be: + * a try-ref after the drop must always be refused. + */ + late = (ctx->released != 0); + + if (nxt_port_use_unless_zero(ctx->port)) { + if (late) { + nxt_atomic_fetch_add(&ctx->late_wins, 1); + } + + nxt_atomic_fetch_add(&ctx->wins, 1); + nxt_atomic_fetch_add(&ctx->port->use_count, -1); + } + + nxt_atomic_fetch_add(&ctx->done, 1); + } +} + + +static nxt_port_t * +nxt_port_uuz_port(nxt_task_t *task) +{ + nxt_port_t *port; + + port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_APP); + + if (nxt_slow_path(port == NULL)) { + return NULL; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + return port; +} + + +/* + * nxt_port_use() cannot be used here: the last drop goes to + * nxt_port_release(), which needs a runtime this test does not build. The + * pool cleanup that nxt_port_new() registered still runs, and still asserts + * the port is quiescent. + */ + +static void +nxt_port_uuz_free(nxt_port_t *port) +{ + port->use_count = 0; + + nxt_mp_destroy(port->mem_pool); +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index d494ca9dd..3821120e9 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -190,6 +190,10 @@ main(int argc, char **argv) return 1; } + if (nxt_port_use_unless_zero_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_mmap_range_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 5f9e668d0..53ef4fb89 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -71,6 +71,7 @@ nxt_int_t nxt_http_chunk_parse_test(nxt_thread_t *thr); nxt_int_t nxt_conf_json_depth_test(nxt_thread_t *thr); nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); +nxt_int_t nxt_port_use_unless_zero_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); diff --git a/test/test_process_shm_oosm.py b/test/test_process_shm_oosm.py index 08314ef6e..b9642f458 100644 --- a/test/test_process_shm_oosm.py +++ b/test/test_process_shm_oosm.py @@ -44,6 +44,8 @@ of work once the PHP worker is warm. """ +import subprocess +import threading import time import pytest @@ -147,3 +149,189 @@ def test_process_shm_oosm(): ) Log.check_alerts() + + +# --------------------------------------------------------------------------- +# The OOSM ack path against an application that is being torn down (issue #195) +# --------------------------------------------------------------------------- + +# Kills, and the gap between them. Each one has to land while the segment is +# still full and clients are still stalled, which is what puts a SHM_ACK +# broadcast and the port teardown of the process it is aimed at on two engines +# at once. +KILLS = 5 +KILL_INTERVAL = 0.4 + +# Stalled readers, so the response buffers -- the application's shared memory +# -- stay busy for the whole run instead of draining between kills. +STALLERS = 3 + +# Bounded on purpose: this is a race driver, not a soak. Five kills at 0.4 s +# is about three seconds of workload, cheap enough for every leg including the +# sanitiser ones. It is timing-dependent by nature: a green run is evidence +# only in the same weak sense as test_process_abrupt_teardown.py, and the +# deterministic check of the primitive this test exercises lives in the C +# suite (src/test/nxt_port_use_unless_zero_test.c). + + +def _oosm_app_pids(): + """PIDs currently serving the application, via ps (as conftest does).""" + out = subprocess.check_output( + ['ps', '-ax', '-o', 'pid', '-o', 'command'] + ).decode() + + marker = f'unit: "{APP}" application' + + return {line.split()[0] for line in out.splitlines() if marker in line} + + +def _wait_for_fresh_pids(spent, timeout=5): + """A worker this run has not killed yet, so every kill is a distinct + abrupt teardown rather than a repeat shot at a corpse. Same reasoning as + test_process_abrupt_teardown.py.""" + + deadline = time.time() + timeout + + while time.time() < deadline: + pids = _oosm_app_pids() - spent + + if pids: + return pids + + time.sleep(0.05) + + return set() + + +def test_process_shm_oosm_kill(skip_alert): + """Kill the OOSM-blocked worker while the router is acknowledging it. + + What this drives: the same one-segment application as above, but with + several clients stalled at once so the segment stays full, and the worker + SIGKILLed underneath them. The router then has two things happening on + two engines: on a worker engine, + ``nxt_port_mmap_buf_completion()`` (src/nxt_port_memory.c) drains a + completed response buffer, wins ``cmp_set(&hdr->oosm, 1, 0)`` and calls + ``nxt_process_broadcast_shm_ack()``; on the router's main thread, the + SIGCHLD-driven ``nxt_port_remove_pid()`` -> ``nxt_process_close_ports()`` + has dropped the app port's references and ``nxt_port_release()`` + (src/nxt_port.c) is destroying its memory pool. + + ``nxt_process_broadcast_shm_ack()`` read the app port out of + ``process->ports`` as a bare pointer and handed it to ``nxt_port_post()``, + whose unconditional ``nxt_atomic_fetch_add(&port->use_count, 1)`` lands on + a port whose count is already zero: the ``port->use_count == 0`` assertion + in ``nxt_port_mp_cleanup()`` in a --debug build, and a use-after-free of + the port and of the posted work item in a release one. The fix takes the + port with ``nxt_port_use_unless_zero()`` under ``rt->processes_mutex``, + which ``nxt_port_release()`` now unlinks under. + + What this proves and what it does not: on a --debug build the assertion + fires as an alert and this test catches it directly -- that is the + reproduction from the issue. On a non-debug build it is only a UAF + driver, worth its runtime under ASan and little without it. A green run + is weak evidence, as in test_process_abrupt_teardown.py; read it as "the + OOSM ack raced repeated abrupt teardowns and the router stayed correct". + """ + + # SIGKILLing a worker is the point of the test, so the main process' + # report of it is expected, not a finding. Nothing else is waived -- + # in particular the assertion alert this test exists to catch is not. + skip_alert(r'process \d+ exited on signal 9') + + client.load(APP, limits={"shm": SHM_LIMIT}, processes=1) + + assert 'success' in client.conf( + {"listen_threads": LISTEN_THREADS}, 'settings' + ), 'listen_threads' + + stop = threading.Event() + failures = [] + + def staller(): + """Ask for a response far larger than the segment and then read it + slowly, so the application blocks in nxt_unit_wait_shm_ack() and the + router keeps completing buffers for it.""" + + while not stop.is_set(): + try: + sock = client.get(url=f'/?mb={RESPONSE_MB}', no_recv=True) + time.sleep(STALL) + client.recvall(sock, read_timeout=READ_TIMEOUT, + buff_size=65536) + sock.close() + + except Exception: + # A request cut short by a kill is the point of the test, not + # a failure. Only a hang or a crashed daemon is, and those + # are caught by the join and the assertions below. + pass + + stallers = [threading.Thread(target=staller) for _ in range(STALLERS)] + + for thread in stallers: + thread.start() + + # Let the first responses reach the OOSM state before killing anything. + time.sleep(STALL * 2) + + killed = [] + + for round_ in range(KILLS): + pids = _wait_for_fresh_pids(set(killed)) + + if not pids: + failures.append(f'no fresh worker for kill {round_ + 1}') + break + + subprocess.call(['kill', '-9', *pids]) + + killed.extend(pids) + + time.sleep(KILL_INTERVAL) + + stop.set() + + for thread in stallers: + thread.join(timeout=120) + assert not thread.is_alive(), 'stalled request thread hung' + + assert not failures, f'{len(failures)} failure(s), first: {failures[0]}' + assert len(killed) >= KILLS, ( + f'only {len(killed)} worker(s) killed: {killed}' + ) + + log = Log.read() + + # The direct hit. nxt_assert() aborts the router in a --debug build, and + # the record it prints names the file and line -- src/nxt_port.c, the + # use_count assertion in nxt_port_mp_cleanup(). Checked before anything + # else, because a dead router makes every other assertion here fail for + # the wrong reason. + assert 'assertion failed' not in log, ( + 'a port was referenced after its last drop: ' + + next( + line + for line in log.splitlines() + if 'assertion failed' in line + ) + ) + + # Still serving, on a worker started after the last kill. + sock = client.get(url='/?mb=1', no_recv=True) + resp = client.recvall(sock, read_timeout=READ_TIMEOUT, buff_size=65536) + sock.close() + + assert _body(resp, 1024 * 1024).count(b'x') == 1024 * 1024, ( + 'still serving' + ) + + if '[debug]' not in log: + pytest.skip('OOSM evidence needs a --debug build') + + # Same proof as the test above that the OOSM round trip really ran; here + # it also establishes that the kills landed on a blocked worker rather + # than an idle one. + assert 'oosm in ' in log, 'router handled an OOSM message' + + Log.check_alerts() From 7a2bb59786c86b027d892d592990f95680db0544 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 21:34:14 +0200 Subject: [PATCH 101/391] php: give the module automount.language_deps mounts PHP was the only language module that declared no mounts, so "isolation.rootfs" left it with nothing bound in from the host. That is not merely inconvenient. A PHP built with the --with-system-tzdata patch -- Debian, Ubuntu and Fedora packages all carry it -- reads the timezone database from the filesystem lazily, at request time. Unit chroot()s in the application worker before nxt_php_start(), so that read resolves inside the rootfs; when the database is not there the patch dereferences NULL instead of raising an error, and the worker dies with SIGSEGV on the application's first date() call. Unit logs only "app process N exited on signal 11". Mirror auto/modules/python and generate nxt__mounts.h with a bind mount for the timezone database directory, and wire it into nxt_app_module. PHP does not expose the path at runtime, so derive it from the configure options php-config reports: take an explicit --with-system-tzdata=DIR as given, probe the usual locations for the bare form Debian and Ubuntu pass, and probe as well when php-config cannot report its options at all, since a spurious mount beats a SIGSEGV. A build with timelib's bundled database gets no mount: it never opens the directory, and mounting one would only add noise to the log of every host that lacks it. That covers the official php:*-cli images and, contrary to what one might expect, Alpine's packages -- 8.4.16 there reports "Olson" 2025.2, not 0.system, and ships no zoneinfo at all. Like the Java, Python and Ruby mounts it is a language dependency, so "automount": {"language_deps": false} opts out of it. Four shapes in the derivation were wrong and are fixed here rather than left for a follow-up: - A literal double quote in the path was taken as shell quoting, so --with-system-tzdata=/opt/a"b mounted /opt/ab, and a later --without-system-tzdata on the same line was swallowed with it. Autoconf quotes with ' only and stock php-config strips those, so a " arrives bare and is an ordinary pathname character. Only ' opens a quoted argument now. - A bare --with-system-tzdata on a build host with no zoneinfo emitted no mount while printing "bundled, no mount required", the opposite of what happened: the crash this commit exists to prevent came back. Fedora 42 passes the option bare exactly as Debian does -- the comment table said otherwise and is corrected -- so this is not a corner case. When the option asked for the probe and nothing is found, mount the patch's compiled-in default and say so. - A path containing ?? made the module build fail: -Wall raises -Wtrigraphs on the emitted literal and -Werror makes it fatal. ? is escaped too. - " and \\ in the path are refused for now, because nxt_discovery_modules() interpolates mount sources into JSON without escaping, and one of either character there leaves every language module unavailable, silently. That is a pre-existing defect, tracked separately; lift this refusal once the serializer escapes. Fixes: freeunitorg/freeunit#256 Co-Authored-By: Claude Opus 5 --- auto/modules/php | 346 +++++++++++++++++++++++++++++++++++++++++++++ src/nxt_php_sapi.c | 6 +- 2 files changed, 350 insertions(+), 2 deletions(-) diff --git a/auto/modules/php b/auto/modules/php index 0e63179d7..d37300651 100644 --- a/auto/modules/php +++ b/auto/modules/php @@ -269,7 +269,352 @@ if grep ^$NXT_PHP_MODULE: $NXT_MAKEFILE 2>&1 > /dev/null; then fi +NXT_PHP_MOUNTS_HEADER=nxt_${NXT_PHP_MODULE}_mounts.h + +# A PHP built with the --with-system-tzdata patch -- Debian, Ubuntu and Fedora +# packages all carry it -- reads the timezone database from the filesystem +# lazily, at request time, that is, after Unit has already chroot()ed into +# "isolation.rootfs". When the database is not there such a build does not +# raise an error, it dereferences NULL: the application worker dies with +# SIGSEGV on the first date() call. Bind mount the database so that PHP gets +# the same automount.language_deps treatment the other language modules get. +# +# PHP does not expose the path at runtime, so derive it here: +# +# --with-system-tzdata=DIR a build that spells the path out; trust it +# even if DIR is absent on the build host. +# --with-system-tzdata Debian, Ubuntu and Fedora pass the flag bare +# and rely on the patch's built-in default, +# /usr/share/zoneinfo: probe for it, and fall +# back to that default when the build host +# has no database at all. +# no such option A build using timelib's bundled database (the +# official php:*-cli images, Alpine's packages); +# it never opens the directory, so mount nothing. +# options unavailable php-config too old to report them: probe, since +# a spurious mount beats a SIGSEGV. +# +# A php-config that does not know --configure-options prints its usage to +# *stdout* and exits nonzero, so the query has to be judged by exit status; +# taking the output alone would read the usage text as a real answer and skip +# the probe. + +# A PHP configured with no options at all reports an empty list and exits +# successfully: that is a bundled-timezonedb build, not an unknown one. Only +# a failed query (a php-config without --configure-options) leaves the +# question open, and that is the one case worth probing the host for. +nxt_php_configure_options_known=yes + +if NXT_PHP_CONFIGURE_OPTIONS=`$NXT_PHP_CONFIG --configure-options 2>/dev/null` +then + NXT_PHP_CONFIGURE_OPTIONS=`printf '%s' "$NXT_PHP_CONFIGURE_OPTIONS" \ + | tr '\n' ' '` +else + NXT_PHP_CONFIGURE_OPTIONS= + nxt_php_configure_options_known=no +fi + +NXT_PHP_TZDATA= +nxt_php_tzprobe=no +nxt_php_tzdata_default=no +nxt_php_tzdata_relative= +nxt_php_tzdata_guessed= + +# Some builds hand back $ac_configure_args verbatim, with each argument +# shell-quoted, and a quoted --with-system-tzdata=DIR may carry a space. +# Split the list into one argument per line the way a shell would -- on +# unquoted whitespace only, quotes removed -- so a quoted argument survives +# whole instead of matching no arm below or being cut at its first space. +# Autoconf quotes with single quotes alone, so a double quote is a pathname +# character here and not quoting: reading it as quoting would swallow the +# rest of the line into one argument and drop the quote from the path. + +nxt_php_configure_args=`printf '%s' "$NXT_PHP_CONFIGURE_OPTIONS" \ + | awk ' + { + n = length($0); q = ""; tok = ""; has = 0; + for (i = 1; i <= n; i++) { + c = substr($0, i, 1); + if (q != "") { + if (c == q) { q = ""; } else { tok = tok c; } + } else if (c == "\047") { + q = c; has = 1; + } else if (c == " " || c == "\t") { + if (has || tok != "") { print tok; } + tok = ""; has = 0; + } else { + tok = tok c; + } + } + if (has || tok != "") { print tok; } + }'` + +# The stock php-config strips the quotes from $configure_options before it +# echoes them (scripts/php-config.in: sed -e "s#'##g"), so a directory with +# whitespace in it arrives already split into words. Recover it: a word +# that starts an absolute tzdata path absorbs the words after it up to the +# next option (-x) or Autoconf variable (NAME=value). It deliberately +# absorbs the positional system type too: no pattern can tell a system +# alias from a path component, since config.sub takes shortened aliases +# ("x86_64-linux") and single words as readily as the cpu-vendor-os +# triple, and a directory name may hold a hyphen just as happily. So this +# pass only has to produce a candidate the true path is a word prefix of; +# the arm below settles which prefix against the build host's filesystem. +# Repeated whitespace is already collapsed by php-config's unquoted echo, +# so it cannot be recovered. + +nxt_php_configure_args=`printf '%s\n' "$nxt_php_configure_args" \ + | awk ' + function flush() { if (path != "") { print path; path = ""; } } + /^--?with-system-tzdata=\// { + flush(); path = $0; next; + } + path != "" && !/^-/ && !/^[A-Za-z_][A-Za-z0-9_]*=/ { + path = path " " $0; next; + } + { flush(); print; } + END { flush(); }'` + +# Walk the arguments in order: Autoconf keeps the last occurrence of a +# repeated option, so a bare --with-system-tzdata appended after an "=no" or +# "=DIR" form is what PHP's ext/date/config.m4 actually saw. It reads the +# value the way Autoconf hands it over: "no" (or --without-system-tzdata) +# disables the patch, "yes" enables it with the built-in default, and only an +# absolute path overrides that default. Every arm therefore clears what the +# others set -- the relative note included, or a later absolute path would be +# mounted under an earlier relative value's "no mount can be derived" notice. + +nxt_php_ifs_saved=$IFS +IFS=' +' +set -f +for nxt_php_opt in $nxt_php_configure_args +do + # Autoconf accepts the single-dash spelling of every option too. + case "$nxt_php_opt" in + -with-* | -without-*) + nxt_php_opt="-$nxt_php_opt" + ;; + esac + + case "$nxt_php_opt" in + --with-system-tzdata | --with-system-tzdata=yes) + NXT_PHP_TZDATA=; nxt_php_tzprobe=yes + nxt_php_tzdata_relative=; nxt_php_tzdata_guessed= + ;; + --with-system-tzdata=no | --without-system-tzdata) + NXT_PHP_TZDATA=; nxt_php_tzprobe=no + nxt_php_tzdata_relative=; nxt_php_tzdata_guessed= + ;; + --with-system-tzdata=/*) + nxt_php_tzprobe=no + nxt_php_tzdata_relative=; nxt_php_tzdata_guessed= + NXT_PHP_TZDATA=`printf '%s' "$nxt_php_opt" \ + | sed 's/^--with-system-tzdata=//'` + + # A candidate carrying whitespace was reassembled from words, + # and only the filesystem knows where the directory ends: take + # the longest word prefix that is one. If no prefix is, keep + # the candidate less a trailing system type and say it was + # reconstructed -- the cross-host case, where the probe loop + # already trusts the build host to look like the rootfs. + case "$NXT_PHP_TZDATA" in + *\ *) + nxt_php_tzcand=$NXT_PHP_TZDATA + + while : + do + if [ -d "$nxt_php_tzcand" ]; then + NXT_PHP_TZDATA=$nxt_php_tzcand + break + fi + + case "$nxt_php_tzcand" in + *\ *) + nxt_php_tzcand=${nxt_php_tzcand% *} + ;; + *) + # Nothing exists. The absorbed tail may + # still be a system type, and dropping a + # trailing alias-shaped word ([a-z0-9_.] + # and at least one hyphen) is the best + # reading left for the message and for a + # rootfs that has what this host lacks. + nxt_php_tzlast=${NXT_PHP_TZDATA##* } + + case "$nxt_php_tzlast" in + *-*) + case "$nxt_php_tzlast" in + *[!a-z0-9_.-]*) ;; + *) + NXT_PHP_TZDATA=${NXT_PHP_TZDATA% *} + ;; + esac + ;; + esac + + nxt_php_tzdata_guessed=$NXT_PHP_TZDATA + break + ;; + esac + done + ;; + esac + ;; + --with-system-tzdata=*) + # A relative value is embedded verbatim and resolved against the + # worker's working directory at run time, so no host directory + # can be derived for a mount; say so instead of mounting an + # unrelated one, and leave the database to the rootfs author. + NXT_PHP_TZDATA=; nxt_php_tzprobe=no + nxt_php_tzdata_guessed= + nxt_php_tzdata_relative=`printf '%s' "$nxt_php_opt" \ + | sed 's/^--with-system-tzdata=//'` + ;; + esac +done +set +f +IFS=$nxt_php_ifs_saved + +if [ $nxt_php_configure_options_known = no ]; then + nxt_php_tzprobe=yes +fi + +if [ $nxt_php_tzprobe = yes ]; then + for nxt_tzdir in /usr/share/zoneinfo /usr/lib/zoneinfo \ + /usr/share/lib/zoneinfo /etc/zoneinfo + do + if [ -f "$nxt_tzdir/UTC" ]; then + NXT_PHP_TZDATA=$nxt_tzdir + break + fi + done + + # When the option itself asked for the probe, PHP does read a system + # database, and the patch compiles /usr/share/zoneinfo in as its + # default -- Debian's and Fedora's php binaries both carry the string. + # A build host without any database (a minimal build chroot, a + # sysroot) must not turn that into "bundled, no mount": emit the + # default, which the worker skips with a warning if the running host + # lacks it too. A probe that only php-config's silence asked for has + # no such certainty, so it still mounts nothing when it finds nothing. + if [ ".$NXT_PHP_TZDATA" = "." ] \ + && [ $nxt_php_configure_options_known = yes ] + then + NXT_PHP_TZDATA=/usr/share/zoneinfo + nxt_php_tzdata_default=yes + fi +fi + +# The path reaches the header as data, never as shell text: escape +# backslash, double quote and -- against -Wtrigraphs, which -Wall raises +# and the default -Werror makes fatal -- the question mark for the C string +# literal, and write the one line that carries it with printf, because an +# unquoted heredoc would re-read "$", "`" and "\" in the path. A control +# character has no representation here -- a newline would break the +# header's line structure as well -- so refuse those. +# +# A double quote and a backslash are refused too, for now: the C literal +# could carry them escaped, but the discovery process hands the mount +# table to the main process as JSON built with a plain %s +# (nxt_discovery_modules(), src/nxt_application.c), where either byte +# breaks the parse and takes every language module down with it. Lift +# this once that serializer escapes its strings. A single quote in the +# path is caught by the same test: the stock php-config's quote strip +# turns Autoconf's '\'' into a bare backslash before it gets here. +nxt_php_tzdata_refused=no + +if [ ".$nxt_php_tzdata_relative" != "." ]; then + printf '%s\n' " + PHP timezone database: relative path \"$nxt_php_tzdata_relative\"" \ + " is resolved against the worker's working directory; no" \ + " mount can be derived, provide it inside the rootfs" + nxt_php_tzdata_refused=yes +fi + +if [ $nxt_php_tzdata_default = yes ]; then + printf '%s\n' " + PHP timezone database: none found on this host;" \ + " mounting the patch's default \"$NXT_PHP_TZDATA\"" +fi + +if [ ".$nxt_php_tzdata_guessed" != "." ] \ + && [ ".$NXT_PHP_TZDATA" = ".$nxt_php_tzdata_guessed" ] +then + printf '%s\n' " + PHP timezone database path \"$NXT_PHP_TZDATA\" was" \ + " reconstructed from a whitespace-split configure option" \ + " and is not a directory here; check it before relying" \ + " on the mount" +fi + +nxt_php_tzdata_printable=$(printf '%s' "$NXT_PHP_TZDATA" \ + | tr -d '\001-\037\177') + +if [ "x$nxt_php_tzdata_printable" != "x$NXT_PHP_TZDATA" ]; then + printf '%s\n' " + PHP timezone database path \"$nxt_php_tzdata_printable\"" \ + " contains control characters that cannot be embedded;" \ + " no mount" + NXT_PHP_TZDATA= + nxt_php_tzdata_refused=yes +fi + +case "$NXT_PHP_TZDATA" in + *\"* | *\\*) + printf '%s\n' " + PHP timezone database path \"$NXT_PHP_TZDATA\"" \ + " contains a double quote or a backslash, which the" \ + " module discovery message cannot carry; no mount" + NXT_PHP_TZDATA= + nxt_php_tzdata_refused=yes + ;; +esac + +if [ ".$NXT_PHP_TZDATA" != "." ]; then + +nxt_php_tzdata_c=$(printf '%s' "$NXT_PHP_TZDATA" | sed 's/[\\"?]/\\&/g') + +cat << 'END' > $NXT_BUILD_DIR/include/$NXT_PHP_MOUNTS_HEADER +#ifndef _NXT_PHP_MOUNTS_H_INCLUDED_ +#define _NXT_PHP_MOUNTS_H_INCLUDED_ + + +static const nxt_fs_mount_t nxt_php_mounts[] = { +END + +printf ' {(u_char *) "%s", (u_char *) "%s",\n' \ + "$nxt_php_tzdata_c" "$nxt_php_tzdata_c" \ + >> $NXT_BUILD_DIR/include/$NXT_PHP_MOUNTS_HEADER + +cat << 'END' >> $NXT_BUILD_DIR/include/$NXT_PHP_MOUNTS_HEADER + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, +}; + +#define NXT_PHP_MOUNTS nxt_php_mounts +#define NXT_PHP_NMOUNTS nxt_nitems(nxt_php_mounts) + + +#endif /* _NXT_PHP_MOUNTS_H_INCLUDED_ */ +END + +else + +cat << 'END' > $NXT_BUILD_DIR/include/$NXT_PHP_MOUNTS_HEADER +#ifndef _NXT_PHP_MOUNTS_H_INCLUDED_ +#define _NXT_PHP_MOUNTS_H_INCLUDED_ + + +#define NXT_PHP_MOUNTS NULL +#define NXT_PHP_NMOUNTS 0 + + +#endif /* _NXT_PHP_MOUNTS_H_INCLUDED_ */ +END + +fi + + $echo " + PHP module: ${NXT_PHP_MODULE}.unit.so" +if [ $nxt_php_tzdata_refused = no ]; then + printf '%s\n' " + PHP timezone database:" \ + " ${NXT_PHP_TZDATA:-bundled, no mount required}" +fi . auto/cc/deps @@ -299,6 +644,7 @@ $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H $NXT_PHP_INCLUDE -DNXT_ZEND_SIGNAL_STARTUP=$NXT_ZEND_SIGNAL_STARTUP \\ -DNXT_PHP_TRUEASYNC=$NXT_PHP_TRUEASYNC \\ -DNXT_PHP_PRE_REQUEST_INIT=$NXT_PHP_PRE_REQUEST_INIT \\ + -DNXT_PHP_MOUNTS_H=\"$NXT_PHP_MOUNTS_HEADER\" \\ $nxt_dep_flags \\ -o $NXT_BUILD_DIR/$nxt_obj $nxt_src $nxt_dep_post diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 65121ec52..4357a4bf7 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -16,6 +16,8 @@ #include #include +#include NXT_PHP_MOUNTS_H + #if (PHP_VERSION_ID >= 50400) #define NXT_HAVE_PHP_IGNORE_CWD 1 @@ -384,8 +386,8 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { compat, nxt_string("php"), PHP_VERSION, - NULL, - 0, + NXT_PHP_MOUNTS, + NXT_PHP_NMOUNTS, nxt_php_setup, nxt_php_start, }; From c2a32bc84eee20f119585bc8c5e885561d995672 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 19:35:08 +0200 Subject: [PATCH 102/391] test(router): assert the listener close precedes the acknowledgement #276 made the router acknowledge a configuration that removes a listener only after the descriptor is closed; #293 replaced the strict bind in test_listeners_port_release with a retry, which guards a leaked descriptor but, as its own commit says, could not have caught #276's regression class. Nothing guarded the ordering. The plan for this (issue #297) was to assert it on the existing debug markers: "engine %p: listen socket close finish" against the controller's "controller conn write". Measured, that does not work. Both orderings read the same on both trees: the reply reaches the controller through a cross-thread post and then a port message to another process, which costs tens of microseconds, while the pre-#276 router's close follows its acknowledgement by two nxt_debug() writes. With #276 reverted, 50 removals idle and 50 under 16x CPU load produced zero inversions of either "close finish" or the raw "socket close(fd)" against the reply. A test written that way would have been green on the very defect it exists to catch. The property is only observable without a race if both records are written by the same thread, and the acknowledgement had no record of its own on the engine that issues it. #297 scopes this to test-only, so the routes that would have kept it there, and why each fails: - The configuration thread's "temp conf %p count" line (nxt_router_conf_ready) is written after the cross-thread post, so its position measures that thread's wake-up latency, not where in the close path the acknowledgement was issued. Same for every controller record: another process again. - "event engine post" (src/nxt_event_engine.c:253) is on the right thread but carries no payload identifying what is being posted, so attributing one to this acknowledgement needs thread-id matching against surrounding records -- fragile, and it would break silently. - src/test/ cannot host it: the property needs a running router with its engines, the controller and a configuration round trip. So add one nxt_debug() to nxt_router_listen_socket_close_ready(), which needs the task passed in. Debug builds only, no release effect. test_listeners_close_before_reply configures a listener, snapshots the log position, removes it, and reads the tail once: the ordering comes from positions within that single read, not from two waits that could each succeed with the order between them wrong. The reply is only the barrier that makes the tail complete -- it is written after every engine has acknowledged, and every engine writes its own records before it acknowledges. The assertion that matters compares the descriptor's close(2) ("socket close(%d)", src/nxt_socket.c:290) with the last acknowledgement. Anchoring on "close finish" instead is not enough: that record is written on entry to nxt_router_listen_socket_close_finish(), before the release, so it cannot see an acknowledgement moved back above nxt_router_listen_socket_release() inside that function -- which is the position this comment describes as the one #276 moved it out of. The close(2) and the acknowledgement are written by the same engine, so the comparison is race-free, and it is red at both positions. There is one record of each kind per engine, so the test compares the last of one with the last of the other. That is independent of how the engines interleave: every engine acknowledges after its own close, so the latest acknowledgement is later than every close; acknowledge first and the latest close is later than every acknowledgement instead. The "socket close(%d)" match is anchored on the router's pid, taken from the close-finish record: the controller writes the same text for its own connection after every reply, and fd numbers collide across processes. All the markers are nxt_debug(), and the suite has no debug-build flag, so the test asks the log for a "[debug]" record and skips naming --debug when there is none. A build that cannot observe the property must not report a pass. Verified against a --debug build in Docker (debian:testing, 8 engines): as committed 50 runs, 50 passed, 0 failed as committed, 16x CPU load 25 runs, 25 passed, 0 failed acknowledge in phase 1 10 runs, 0 passed, 10 failed acknowledge above the release 10 runs, 0 passed, 10 failed built without --debug 1 run, skipped Both close paths hold at listen_threads 1 and 32 as well as the default. Co-Authored-By: Claude Opus 5 --- src/nxt_router.c | 25 +++++++++-- test/test_configuration.py | 86 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 6d6ad31dc..1c8395414 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -4153,7 +4153,8 @@ nxt_router_worker_thread_exit(nxt_task_t *task) * pattern in nxt_router_worker_thread_quit()). */ -static void nxt_router_listen_socket_close_ready(nxt_socket_conf_joint_t *joint); +static void nxt_router_listen_socket_close_ready(nxt_task_t *task, + nxt_socket_conf_joint_t *joint); static void nxt_router_listen_socket_close_finish(nxt_task_t *task, nxt_listen_event_t *lev); @@ -4226,7 +4227,7 @@ nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "engine %p: listen socket %d drain pending, " "in-flight: %D", engine, lev->socket.fd, lev->count - 1); - nxt_router_listen_socket_close_ready(joint); + nxt_router_listen_socket_close_ready(task, joint); return; } @@ -4236,7 +4237,8 @@ nxt_router_listen_socket_close(nxt_task_t *task, void *obj, void *data) static void -nxt_router_listen_socket_close_ready(nxt_socket_conf_joint_t *joint) +nxt_router_listen_socket_close_ready(nxt_task_t *task, + nxt_socket_conf_joint_t *joint) { nxt_joint_job_t *job; @@ -4247,6 +4249,21 @@ nxt_router_listen_socket_close_ready(nxt_socket_conf_joint_t *joint) joint->close_job = NULL; + /* + * The only record of the acknowledgement on the thread that issues + * it. The configuration thread's "temp conf ... count" line is + * written after a cross-thread post, so where it lands in the log is + * a function of that thread's wake-up latency, not of where in the + * close path the acknowledgement was issued -- it stays in the same + * place whether this call is made before the descriptor is released + * or after it. This line does not: it is written by the closing + * engine itself, between its own records, so the order of the close + * and the acknowledgement is readable from the log. + * test_listeners_close_before_reply asserts exactly that. + */ + nxt_debug(task, "engine %p: listen socket close acknowledged", + task->thread->engine); + nxt_router_conf_wait_post(job); } @@ -4290,7 +4307,7 @@ nxt_router_listen_socket_close_finish(nxt_task_t *task, * nxt_router_listen_socket_release() frees the nxt_listen_socket_t, * not the socket configuration the joint points at. */ - nxt_router_listen_socket_close_ready(joint); + nxt_router_listen_socket_close_ready(task, joint); nxt_router_listen_event_release(task, lev, joint); } diff --git a/test/test_configuration.py b/test/test_configuration.py index 0283357a5..8342cbf14 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -1,9 +1,11 @@ +import re import socket import time import pytest from unit.control import Control +from unit.log import Log prerequisites = {'modules': {'python': 'any'}} @@ -372,6 +374,90 @@ def test_listeners_port_release(): assert 'success' in resp, 'port release' +def test_listeners_close_before_reply(findall): + assert 'success' in client.conf( + { + "listeners": {"127.0.0.1:8080": {"pass": "routes"}}, + "routes": [], + } + ), 'listener added' + + # The markers below are all nxt_debug(), so a release build cannot + # observe this at all. There is no build flag in option.available, + # so ask the log: skipping is the only honest outcome here, passing + # would be a test that cannot fail. + if not findall(r'\[debug\]'): + pytest.skip('needs a build configured with --debug') + + # Remember where the log ends before the removal. The tail already + # holds a "controller conn write" for the request above, and the + # ordering is read off positions in a single read of what follows: + # two independent waits could each succeed with the order between + # them wrong. + pos = len(Log.read()) + + assert 'success' in client.conf( + {"listeners": {}, "applications": {}} + ), 'listener removed' + + # The reply is the barrier that makes the tail complete rather than + # something the assertions below compare against: it is written only + # after every engine has acknowledged, and every engine writes its + # own records before it acknowledges. + deadline = time.monotonic() + 15 + + while True: + tail = Log.read()[pos:] + + if 'controller conn write complete' in tail: + break + + if time.monotonic() >= deadline: + pytest.fail('the controller did not reply') + + time.sleep(0.01) + + # One record per engine, so with listen_threads > 1 there are + # several of each. Comparing the last of one with the last of the + # other is what makes the assertion independent of how the engines + # interleave: an engine acknowledges after it has finished closing, + # so the latest acknowledgement is later than every close; an engine + # that acknowledged first would put the latest close after every + # acknowledgement instead. + finish = list( + re.finditer( + r'\[debug\] (\d+)#\d+ .*listen socket close finish: (\d+)', + tail, + ) + ) + ack = list(re.finditer(r'listen socket close acknowledged', tail)) + + assert finish, 'listener closed' + assert ack, 'removal acknowledged' + + assert finish[-1].start() < ack[-1].start(), 'phase 2 before ack' + + # The one that has to hold: "close finish" is written on entry to + # nxt_router_listen_socket_close_finish(), before the descriptor is + # released, so the check above cannot see an acknowledgement moved + # back above nxt_router_listen_socket_release() inside that function + # -- the position #276 moved it out of. The close(2) of the + # descriptor is written by the same engine that then acknowledges, + # so anchoring on it covers both positions. + # + # Match the router's pid: the controller writes "socket close(N)" + # for its own connection after every reply, and fd numbers collide + # across processes. + pid, fd = finish[-1].group(1), finish[-1].group(2) + closed = re.search( + rf'\[debug\] {pid}#\d+ (\*\d+ )?socket close\({fd}\)', tail + ) + + assert closed is not None, 'descriptor closed' + + assert closed.start() < ack[-1].start(), 'closed before ack' + + def test_json_application_name_large(): name = "X" * 1024 * 1024 From 826fba01cd8226c1e50f5634851559827c0ce405 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 21:12:21 +0200 Subject: [PATCH 103/391] ci(sanitize): run test_configuration.py in the python leg test_listeners_close_before_reply reads nxt_debug() records, so it skips on any build without --debug. Every pytest leg of build-test.yml is a release build (its single configure step passes no --debug), and the one workflow that runs pytest against a --debug build, sanitize.yml, runs a fixed list of files that did not include test_configuration.py. The ordering regression the test exists to catch could therefore merge with every check green, the test skipping in each leg that ran it and absent from the only leg in which it would run. Add the file to the python leg's list. A separate --debug leg would build and configure a second time for one test that takes a second; the python leg already has the right build, no sudo, and --restart, which the test does not need but tolerates. Making the assertion observable in a release build would mean promoting the markers out of nxt_debug(), which the test's own commit rules out. The file brings one new UBSan site into the leg: nxt_is_tstr() calls memchr() on a zero-length nxt_str_t whose .start is NULL, the class freeunitorg/freeunit#200 already tracks for memcmp() and memcpy(). It is reached from the configuration validator, which no other file in the sanitizer legs drives with the configurations that get there. Waive it pinned to src/nxt_tstr.h under #200, as the two existing entries of that class are. Measured locally on the leg's exact build and invocation (--debug, ASan+UBSan, -v --save-log --restart): --collect-only lists the test under test_configuration.py; the file runs 35 passed, 4 skipped in 20s, the four skips being the file's pre-existing "not yet" marks, and the test PASSED. With this PR's nxt_router.c hunk reverted it fails on "removal acknowledged"; with the marker kept but the acknowledgement moved back before nxt_router_listen_socket_release() it fails on "closed before ack". The gate script passes with the new entry and fails without it. Co-Authored-By: Claude Opus 5 --- .github/sanitizer-allowlist.txt | 7 +++++++ .github/workflows/sanitize.yml | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/sanitizer-allowlist.txt b/.github/sanitizer-allowlist.txt index b09bddddd..9e37d8ad1 100644 --- a/.github/sanitizer-allowlist.txt +++ b/.github/sanitizer-allowlist.txt @@ -30,3 +30,10 @@ src/nxt_http_route\.c:[0-9]+:[0-9]+: runtime error: null pointer passed as argum # freeunitorg/freeunit#200 -- same class in nxt_cpymem(): memcpy(dst, NULL, 0). src/nxt_string\.h:[0-9]+:[0-9]+: runtime error: null pointer passed as argument 2, which is declared to never be null + +# freeunitorg/freeunit#200 -- same class in nxt_is_tstr(): memchr(NULL, c, 0) +# on a zero-length nxt_str_t, twice, one per character it looks for. Reached +# from the configuration validator. Measured with the python leg's test set: +# only test_configuration.py sends a configuration that gets there, so the +# entry arrived with that file. +src/nxt_tstr\.h:[0-9]+:[0-9]+: runtime error: null pointer passed as argument 1, which is declared to never be null diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 0ff58a9b0..1a2616eea 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -131,6 +131,14 @@ jobs: # hard ASan failure here instead of an intermittent "[alert]" line # in one runtime's log. It self-skips every runtime that is not # built into this leg's unitd. + # + # test_configuration drives the control API only and is here for + # test_listeners_close_before_reply: it reads the router's + # nxt_debug() records to assert that a removed listener's + # descriptor is closed before the removal is acknowledged + # (#276), and skips on any build without --debug. Every + # build-test leg is a release build, so this leg is the only one + # in which that test runs rather than skips. tests: >- test/test_graceful_reload.py test/test_listener_drain.py @@ -140,6 +148,7 @@ jobs: test/test_chunked.py test/test_conn_recycle.py test/test_app_lifecycle.py + test/test_configuration.py - runtime: php apt: "" configure: php From b10d89f793d9bf4c3223b620209d50a491518cc0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 8 Sep 2026 21:18:34 +0200 Subject: [PATCH 104/391] discovery: escape the module list as JSON nxt_discovery_modules() formatted the module list it sends to the main process with nxt_sprintf() and "%s": a double quote or a backslash in a module's path or in a mount source produced invalid JSON, the main process's nxt_conf_json_parse() failed, and it went on to start the controller and the router with no language modules at all, logging nothing above debug level. Every application then failed with "The module to run ... is not found among the available application modules". Reproduced on master with --modulesdir '/tmp/mod"dir': the message carried "file": "/tmp/mod"dir/php.unit.so" and the PHP module was gone; the same happens for any language's mount table, which is how #265's tzdata path surfaced it. Build the message through nxt_conf and print it with nxt_conf_json_print(), which escapes every string and sizes the buffer itself. The member names and types are unchanged, so the main process's nxt_app_lang_module_map and nxt_app_lang_mounts_map read it as before. With the fix the same modulesdir yields "file":"/tmp/mod\"dir/..." and the application serves 200. Co-Authored-By: Claude Opus 5 --- src/nxt_application.c | 147 ++++++++++++++++++++++++----------------- src/nxt_main_process.c | 7 ++ 2 files changed, 93 insertions(+), 61 deletions(-) diff --git a/src/nxt_application.c b/src/nxt_application.c index af61dbc50..8d59663ce 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -225,17 +225,28 @@ nxt_discovery_start(nxt_task_t *task, nxt_process_data_t *data) static nxt_buf_t * nxt_discovery_modules(nxt_task_t *task, const char *path) { - char *name; - u_char *p, *end; - size_t size; - glob_t glb; - nxt_mp_t *mp; - nxt_buf_t *b; - nxt_int_t ret; - nxt_uint_t i, n, j; - nxt_array_t *modules, *mounts; - nxt_module_t *module; - nxt_fs_mount_t *mnt; + char *name; + size_t size; + glob_t glb; + nxt_mp_t *mp; + nxt_str_t str; + nxt_buf_t *b; + nxt_int_t ret; + nxt_uint_t i, n, j; + nxt_array_t *modules, *mounts; + nxt_module_t *module; + nxt_fs_mount_t *mnt; + nxt_conf_value_t *root, *obj, *array, *mount; + + static const nxt_str_t type_str = nxt_string("type"); + static const nxt_str_t name_str = nxt_string("name"); + static const nxt_str_t version_str = nxt_string("version"); + static const nxt_str_t file_str = nxt_string("file"); + static const nxt_str_t mounts_str = nxt_string("mounts"); + static const nxt_str_t src_str = nxt_string("src"); + static const nxt_str_t dst_str = nxt_string("dst"); + static const nxt_str_t flags_str = nxt_string("flags"); + static const nxt_str_t data_str = nxt_string("data"); b = NULL; @@ -268,86 +279,100 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) } } - size = nxt_length("[]"); module = modules->elts; n = modules->nelts; + /* + * The main process parses this message as JSON, so build it through + * nxt_conf, which escapes every string. Formatted with "%s", a double + * quote or a backslash in a module path or a mount source broke the + * parse, and the main process then started with no language modules + * at all and nothing above debug level in the log to say why. + */ + root = nxt_conf_create_array(mp, n); + if (nxt_slow_path(root == NULL)) { + goto fail; + } + for (i = 0; i < n; i++) { nxt_debug(task, "module: %d %V %V", module[i].type, &module[i].version, &module[i].file); - size += nxt_length("{\"type\": ,"); - size += nxt_length(" \"name\": \"\","); - size += nxt_length(" \"version\": \"\","); - size += nxt_length(" \"file\": \"\","); - size += nxt_length(" \"mounts\": []},"); + obj = nxt_conf_create_object(mp, 5); + if (nxt_slow_path(obj == NULL)) { + goto fail; + } - size += NXT_INT_T_LEN - + module[i].version.length - + module[i].name.length - + module[i].file.length; + nxt_conf_set_member_integer(obj, &type_str, module[i].type, 0); + nxt_conf_set_member_string(obj, &name_str, &module[i].name, 1); + nxt_conf_set_member_string(obj, &version_str, &module[i].version, 2); + nxt_conf_set_member_string(obj, &file_str, &module[i].file, 3); mounts = module[i].mounts; - - size += mounts->nelts * nxt_length("{\"src\": \"\", \"dst\": \"\", " - "\"type\": , \"name\": \"\", " - "\"flags\": , \"data\": \"\"},"); - mnt = mounts->elts; - for (j = 0; j < mounts->nelts; j++) { - size += nxt_strlen(mnt[j].src) + nxt_strlen(mnt[j].dst) - + nxt_strlen(mnt[j].name) + (2 * NXT_INT_T_LEN) - + (mnt[j].data == NULL ? 0 : nxt_strlen(mnt[j].data)); + array = nxt_conf_create_array(mp, mounts->nelts); + if (nxt_slow_path(array == NULL)) { + goto fail; } - } - b = nxt_buf_mem_alloc(mp, size, 0); - if (b == NULL) { - goto fail; - } + for (j = 0; j < mounts->nelts; j++) { + mount = nxt_conf_create_object(mp, 6); + if (nxt_slow_path(mount == NULL)) { + goto fail; + } - b->completion_handler = nxt_discovery_completion_handler; + str.start = mnt[j].src; + str.length = nxt_strlen(mnt[j].src); + nxt_conf_set_member_string(mount, &src_str, &str, 0); - p = b->mem.free; - end = b->mem.end; - *p++ = '['; + str.start = mnt[j].dst; + str.length = nxt_strlen(mnt[j].dst); + nxt_conf_set_member_string(mount, &dst_str, &str, 1); - for (i = 0; i < n; i++) { - mounts = module[i].mounts; + str.start = mnt[j].name; + str.length = nxt_strlen(mnt[j].name); + nxt_conf_set_member_string(mount, &name_str, &str, 2); - p = nxt_sprintf(p, end, "{\"type\": %d, \"name\": \"%V\", " - "\"version\": \"%V\", \"file\": \"%V\", \"mounts\": [", - module[i].type, &module[i].name, &module[i].version, - &module[i].file); + nxt_conf_set_member_integer(mount, &type_str, mnt[j].type, 3); + nxt_conf_set_member_integer(mount, &flags_str, mnt[j].flags, 4); - mnt = mounts->elts; - for (j = 0; j < mounts->nelts; j++) { - p = nxt_sprintf(p, end, - "{\"src\": \"%s\", \"dst\": \"%s\", " - "\"name\": \"%s\", \"type\": %d, \"flags\": %d, " - "\"data\": \"%s\"},", - mnt[j].src, mnt[j].dst, mnt[j].name, mnt[j].type, - mnt[j].flags, - mnt[j].data == NULL ? (u_char *) "" : mnt[j].data); + str.start = (mnt[j].data == NULL) ? (u_char *) "" : mnt[j].data; + str.length = nxt_strlen(str.start); + nxt_conf_set_member_string(mount, &data_str, &str, 5); + + nxt_conf_set_element(array, j, mount); } - *p++ = ']'; - *p++ = '}'; - *p++ = ','; + nxt_conf_set_member(obj, &mounts_str, array, 4); + nxt_conf_set_element(root, i, obj); } - *p++ = ']'; + size = nxt_conf_json_length(root, NULL); - if (nxt_slow_path(p > end)) { - nxt_alert(task, "discovery write past the buffer"); + b = nxt_buf_mem_alloc(mp, size, 0); + if (b == NULL) { goto fail; } - b->mem.free = p; + b->completion_handler = nxt_discovery_completion_handler; + + b->mem.free = nxt_conf_json_print(b->mem.free, root, NULL); fail: + /* + * This is the success path too -- b is NULL only if one of the allocations + * above failed. Say so: the caller turns a NULL into NXT_ERROR, and the + * main process then starts the controller and the router with no language + * modules registered, so without a line here the only symptom an operator + * sees is every application type reported as "not found". + */ + if (nxt_slow_path(b == NULL)) { + nxt_alert(task, "discovery failed to build the module list"); + nxt_mp_destroy(mp); + } + globfree(&glb); return b; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 027211977..56c95cd8b 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -1705,11 +1705,15 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) conf = nxt_conf_json_parse(mp, b->mem.pos, b->mem.free, NULL); if (conf == NULL) { + nxt_alert(task, "discovery message is not valid JSON; " + "no application modules will be available"); goto fail; } root = nxt_conf_get_path(conf, &root_path); if (root == NULL) { + nxt_alert(task, "discovery message has no module list; " + "no application modules will be available"); goto fail; } @@ -1721,6 +1725,7 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) lang = nxt_array_zero_add(rt->languages); if (lang == NULL) { + nxt_alert(task, "failed to record the module at index %uD", index); goto fail; } @@ -1730,6 +1735,8 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_nitems(nxt_app_lang_module_map), lang); if (ret != NXT_OK) { + nxt_alert(task, "unexpected members in the module at index %uD", + index); goto fail; } From 9c0ab65c73f29770c82cd98b115e011cd867f865 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 00:00:02 +0200 Subject: [PATCH 105/391] main: store the state files atomically nxt_main_file_store() (src/nxt_main_process.c:1924) took a tmp_name argument and never used it. It set file.name from "name", opened the destination itself with NXT_FILE_TRUNCATE, wrote into it, and finished with nxt_file_rename(file.name, name) -- a rename of the file onto itself. conf.json and version were therefore replaced in place, with a window in which the destination was truncated to zero and only partly rewritten; a crash, a power loss or an ENOSPC inside that window left a truncated state file. Restoring one degrades to an empty configuration with nothing but a log alert (src/nxt_controller.c:196), so an interrupted store could silently bring the daemon back serving nothing. The short-write path was worse than the crash: it did nxt_file_delete(file.name), and under that spelling file.name was the live conf.json. A store that ran out of space removed the existing good configuration outright, no crash required. The pytest added here observes exactly that on the unfixed code -- with the state directory on a 64 KiB tmpfs, a configuration too large to store leaves no conf.json at all. nxt_main_file_store() now writes tmp_name, fsyncs it, renames it over the destination and fsyncs the directory that holds them; every failure before the rename unlinks the temporary and leaves the destination untouched. The directory flush is what makes the rename itself survive a power loss, and its failure is not treated as a store failure since the destination is already in place. The two callers (src/nxt_main_process.c:1892 and :1900) pass rt->state, the directory both paths are built from (src/nxt_runtime.c:927-961); the temporary names they already passed are unchanged. Because the replacement now arrives by rename(2), a conf.json or version that was a symlink is replaced by a regular file rather than followed. That is the price of atomicity and is the behaviour the temporary-file plumbing always intended. A symlinked state *directory* is unaffected: rt->state, the temporary and the destination all resolve through it alike, so the temporary is still created beside the file it replaces. nxt_file_sync() and nxt_file_dir_sync() are new in src/nxt_file.c; the directory flush tolerates EINVAL and EOPNOTSUPP, which some filesystems return for fsync on a directory descriptor. The failure is now diagnosable. The nxt_file_t was memzero'd, so log_level was 0 -- the "say nothing" value -- and nxt_file_open() reported nothing; NXT_LOG_ERR is the loudest level that is not zero and is set before the open. A short pwrite() carries no errno and so was silent even in nxt_file_write(); it is logged with both byte counts. On the tmpfs above the store now says [alert] write("/.../conf.json.tmp") stored 57344 of 99543 bytes [alert] failed to store current configuration Mode and ownership are preserved: the temporary is created 0600 as before, and when the destination already exists its mode and owner are carried onto the replacement, so a state file an administrator has re-permissioned is not reverted. Main runs as root, so the fchown() normally succeeds; a failure is logged and the store proceeds, since refusing to persist the configuration would be the worse outcome. The temporary is not opened with O_TRUNC. Its name is predictable and the store runs as root, so a --statedir writable by any other user is a place to leave a symbolic link under that name: O_TRUNC would follow it and truncate whatever it addresses, and the rename would then install the link as the state file. Unlink first and create with O_CREAT|O_EXCL, which refuses a link outright -- and refuses a regular file somebody hard-linked to a target we must not write, which O_NOFOLLOW alone would not. The unlink is what replaces a leftover temporary from an interrupted store; the window between the two is not a hole, since something recreated in it makes the create fail, losing the store and keeping the target rather than the other way round. nxt_file_delete() is not used for that unlink because it logs an alert on every failure, ENOENT included -- and no leftover is the ordinary case. Ownership is carried over before the mode. A chown() of a regular file clears set-user-ID, so the other order drops exactly the bit "fi.st_mode & 07777" is there to preserve. The destination is read with lstat(), not stat(), and donates nothing unless it is a regular file. Under the same threat model that makes the temporary exclusive -- a state directory another user can write -- that user can also replace conf.json with a symbolic link to a file of their own. stat() follows it, so the replacement would inherit that file's owner and mode: aim the link at something 0666 and conf.json comes back world-writable, and stays so, because every later store inherits it again. A destination that is not a plain regular file now leaves the temporary on the 0600 it was created with. On the path the inheritance exists for -- an administrator's re-permissioned state file -- lstat() and stat() agree. A destination that cannot be read at all no longer fails the store: the temporary is already written and 0600 is serviceable, so an unreadable destination costs the inheritance rather than the configuration, which is the trade the fchown() beside it already makes. The write loops. pwrite() may store less than it was asked for, and a change whose subject is surviving a partial store should resume from where it stopped rather than turn a short write into a failed one. src/test/nxt_main_file_store_test.c drives the store through a new NXT_TESTS wrapper and pins the invariants down: the destination's inode changes across a store (an in-place rewrite keeps it), no temporary survives -- including a stale one planted in the way, a store whose temporary cannot be created reports NXT_ERROR and leaves the destination byte-for-byte intact, and an existing 0640 destination comes back 0640. Both it and test/test_state_store.py fail on the unfixed function. Three cases cover the hardening, each run against the code without it first: a temporary that is a symbolic link reports "the store followed the link and wrote through it"; a destination whose mode is 04640 comes back 0x1a0 instead of 0x9a0 when the chown() follows the chmod(); and a destination that is a symbolic link to a 0666 file comes back 0x1b6 rather than the 0600 the temporary was created with. The set-user-ID case needs no root -- an unchanged owner is enough for the kernel to clear the bit. The store now fsync()s the state file and the directory holding it on the main process's event loop, which is also the loop that supervises every child and answers every port message. On a slow or loaded device an accepted PUT /config therefore adds a stall there that it did not have before, and process starts queue behind it. That is the cost of the durability this change is for, and it is named here rather than left to be discovered: moving the store off that loop is a larger change than making it atomic. test/syscalls/capture.sh issues a second PUT. With one, the state directory is fresh, nxt_main_file_store_inherit() finds no destination and returns before reading anything, and the fchmod() and fchown() that carry a destination's mode and ownership onto its replacement -- the most privileged part of this change -- were invisible to the drift gate. The baselines are refreshed from a capture of the second PUT rather than by hand, and the two libcs do not agree: both gain fchmod and fchown and lose the bare stat, and musl additionally reports lstat where glibc decodes the same call as newfstatat, which it already had. Not addressed here: the controller answers the configuration request before the store is attempted, because CONF_STORE carries no reply (src/nxt_controller.c:604-663). A store that fails is therefore still reported to the client as a success, and only the log says otherwise. Threading a reply back through that path is a separate change. Fixes #215 Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + src/nxt_file.c | 59 ++++ src/nxt_file.h | 44 +++ src/nxt_main_process.c | 214 ++++++++++++- src/nxt_main_process.h | 2 + src/test/nxt_main_file_store_test.c | 421 +++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/syscalls/baseline-linux-glibc.txt | 3 + test/syscalls/baseline-linux-musl.txt | 4 + test/syscalls/capture.sh | 17 + test/test_state_store.py | 141 +++++++++ 12 files changed, 899 insertions(+), 12 deletions(-) create mode 100644 src/test/nxt_main_file_store_test.c create mode 100644 test/test_state_store.py diff --git a/auto/sources b/auto/sources index 0526baebc..c2d57edf8 100644 --- a/auto/sources +++ b/auto/sources @@ -186,6 +186,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_proto_death_test.c \ src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_main_start_process_reply_test.c \ + src/test/nxt_main_file_store_test.c \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_fd_test.c \ " diff --git a/src/nxt_file.c b/src/nxt_file.c index d29695e79..34a551ce5 100644 --- a/src/nxt_file.c +++ b/src/nxt_file.c @@ -426,6 +426,65 @@ nxt_file_rename(nxt_file_name_t *old_name, nxt_file_name_t *new_name) } +nxt_int_t +nxt_file_sync(nxt_task_t *task, nxt_file_t *file) +{ + nxt_debug(task, "fsync(%FD, \"%FN\")", file->fd, file->name); + + if (nxt_fast_path(fsync(file->fd) == 0)) { + return NXT_OK; + } + + file->error = nxt_errno; + + nxt_alert(task, "fsync(%FD, \"%FN\") failed %E", + file->fd, file->name, file->error); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_file_dir_sync(nxt_task_t *task, nxt_file_name_t *name) +{ + int fd; + nxt_int_t ret; + nxt_err_t err; + + fd = open((char *) name, O_RDONLY | NXT_FILE_DIR_SYNC_FLAGS); + if (nxt_slow_path(fd == -1)) { + nxt_alert(task, "open(\"%FN\", O_RDONLY) failed %E", name, nxt_errno); + return NXT_ERROR; + } + + nxt_debug(task, "fsync(%FD, \"%FN\") directory", fd, name); + + ret = NXT_OK; + + if (nxt_slow_path(fsync(fd) != 0)) { + err = nxt_errno; + + /* + * Some file systems refuse to flush a directory descriptor at all; + * the rename() has still happened, so this is not a store failure. + */ + if (err == NXT_EINVAL || err == NXT_EOPNOTSUPP) { + nxt_debug(task, "fsync(%FD, \"%FN\") directory unsupported %E", + fd, name, err); + + } else { + nxt_alert(task, "fsync(%FD, \"%FN\") directory failed %E", + fd, name, err); + ret = NXT_ERROR; + } + } + + nxt_fd_close(fd); + + return ret; +} + + /* * ioctl(FIONBIO) sets a non-blocking mode using one syscall, * thereas fcntl(F_SETFL, O_NONBLOCK) needs to learn the current state diff --git a/src/nxt_file.h b/src/nxt_file.h index 0c03a5f90..aa9a3181d 100644 --- a/src/nxt_file.h +++ b/src/nxt_file.h @@ -141,11 +141,44 @@ NXT_EXPORT nxt_int_t nxt_file_openat2(nxt_task_t *task, nxt_file_t *file, #endif /* NXT_HAVE_OPENAT2 */ +/* + * O_DIRECTORY, where the platform has it, keeps nxt_file_dir_sync() from + * opening something that is not a directory. It is not in POSIX, so the + * flag is optional. + */ +#if defined(O_DIRECTORY) +#define NXT_FILE_DIR_SYNC_FLAGS O_DIRECTORY +#else +#define NXT_FILE_DIR_SYNC_FLAGS 0 +#endif + + +/* + * O_NOFOLLOW, where the platform has it, keeps a create from opening a + * symbolic link left in place of the file. O_EXCL already refuses one -- + * POSIX has open() fail when O_CREAT and O_EXCL are set and the path names + * a link -- so this is the belt to that pair of braces, and optional. + */ +#if defined(O_NOFOLLOW) +#define NXT_FILE_NOFOLLOW O_NOFOLLOW +#else +#define NXT_FILE_NOFOLLOW 0 +#endif + + /* The file creation modes. */ #define NXT_FILE_CREATE_OR_OPEN O_CREAT #define NXT_FILE_OPEN 0 #define NXT_FILE_TRUNCATE (O_CREAT | O_TRUNC) +/* + * Create the file or fail: never open something that is already there, and + * never follow a link to it. For a path an unprivileged user could have + * got to first -- see nxt_main_file_store(). + */ +#define NXT_FILE_CREATE_EXCLUSIVE \ + (O_CREAT | O_EXCL | NXT_FILE_NOFOLLOW) + /* The file access rights. */ #define NXT_FILE_DEFAULT_ACCESS 0644 #define NXT_FILE_OWNER_ACCESS 0600 @@ -182,6 +215,17 @@ NXT_EXPORT nxt_int_t nxt_file_chown(nxt_file_name_t *name, const char *owner, NXT_EXPORT nxt_int_t nxt_file_rename(nxt_file_name_t *old_name, nxt_file_name_t *new_name); +/* + * Flush an open file, respectively a directory named by path, to stable + * storage. A durable replace is: write the temporary file, nxt_file_sync() + * it, nxt_file_rename() it over the destination, then nxt_file_dir_sync() + * the directory that holds them -- without the last step the rename itself + * may not survive a power loss. + */ +NXT_EXPORT nxt_int_t nxt_file_sync(nxt_task_t *task, nxt_file_t *file); +NXT_EXPORT nxt_int_t nxt_file_dir_sync(nxt_task_t *task, + nxt_file_name_t *name); + NXT_EXPORT nxt_int_t nxt_fd_nonblocking(nxt_task_t *task, nxt_fd_t fd); NXT_EXPORT nxt_int_t nxt_fd_blocking(nxt_task_t *task, nxt_fd_t fd); NXT_EXPORT ssize_t nxt_fd_write(nxt_fd_t fd, u_char *buf, size_t size); diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 56c95cd8b..c5c5b59bf 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -62,8 +62,10 @@ static void nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); -static nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *tmp_name, - const char *name, u_char *buf, size_t size); +static nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *dir, + const char *tmp_name, const char *name, u_char *buf, size_t size); +static nxt_int_t nxt_main_file_store_inherit(nxt_task_t *task, + nxt_file_t *tmp, const char *name); static void nxt_main_port_access_log_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); @@ -731,6 +733,20 @@ nxt_main_test_run_start_process_handler(nxt_task_t *task, nxt_main_start_process_handler(task, msg); } + +/* + * Public wrapper that lets src/test/nxt_main_file_store_test.c drive the + * static nxt_main_file_store() against a scratch directory -- used to + * verify that the store is atomic and never damages the existing file + * (issue #215). + */ +nxt_int_t +nxt_main_test_run_file_store(nxt_task_t *task, const char *dir, + const char *tmp_name, const char *name, u_char *buf, size_t size) +{ + return nxt_main_file_store(task, dir, tmp_name, name, buf, size); +} + #endif @@ -1896,7 +1912,7 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_conf_ver != NXT_VERNUM) { n = nxt_sprintf(ver, ver + NXT_INT_T_LEN, "%d", NXT_VERNUM) - ver; - ret = nxt_main_file_store(task, rt->ver_tmp, rt->ver, ver, n); + ret = nxt_main_file_store(task, rt->state, rt->ver_tmp, rt->ver, ver, n); if (nxt_slow_path(ret != NXT_OK)) { goto error; } @@ -1904,7 +1920,7 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_conf_ver = NXT_VERNUM; } - ret = nxt_main_file_store(task, rt->conf_tmp, rt->conf, p, size); + ret = nxt_main_file_store(task, rt->state, rt->conf_tmp, rt->conf, p, size); if (nxt_fast_path(ret == NXT_OK)) { goto cleanup; @@ -1927,34 +1943,208 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Replace "name" with "size" bytes of "buf" atomically: the content goes to + * "tmp_name", which the caller places in "dir" beside the destination, is + * flushed, and only then rename(2)d over "name", so a reader sees either + * the whole old file or the whole new one. "dir" is flushed after the + * rename, without which the rename may not survive a power loss. Every + * failure before the rename unlinks the temporary and leaves the existing + * "name" untouched. + * + * Because the replacement arrives by rename(), a "name" that was a symlink + * is replaced by a regular file rather than followed -- the price of + * atomicity. A symlinked state *directory* is unaffected: "dir" and + * "tmp_name" resolve through it just as "name" does. + * + * The temporary is created 0600 as before; when the destination already + * exists, its mode and ownership are carried over to the replacement, so a + * state file an administrator has re-permissioned keeps its settings. + */ static nxt_int_t -nxt_main_file_store(nxt_task_t *task, const char *tmp_name, const char *name, - u_char *buf, size_t size) +nxt_main_file_store(nxt_task_t *task, const char *dir, const char *tmp_name, + const char *name, u_char *buf, size_t size) { + size_t written; ssize_t n; nxt_int_t ret; nxt_file_t file; nxt_memzero(&file, sizeof(nxt_file_t)); - file.name = (nxt_file_name_t *) name; + file.name = (nxt_file_name_t *) tmp_name; - ret = nxt_file_open(task, &file, NXT_FILE_WRONLY, NXT_FILE_TRUNCATE, - NXT_FILE_OWNER_ACCESS); + /* + * Without a log level nxt_file_open() reports nothing, and a failed + * store would be diagnosable only from the caller's summary alert. + * NXT_LOG_ALERT is zero, which is the "say nothing" value, so the + * loudest usable level here is NXT_LOG_ERR. + */ + file.log_level = NXT_LOG_ERR; + + /* + * Unlink first, then create exclusively, rather than opening the + * temporary with O_TRUNC. The name is predictable and the store runs + * as root, so if --statedir is writable by anyone else that user can + * put a symbolic link there ahead of us: O_TRUNC would follow it and + * truncate whatever it addresses, and the rename below would then + * install the link as the state file. O_EXCL refuses a link outright + * and refuses a regular file somebody hard-linked to a target we must + * not write, which O_NOFOLLOW alone would not. + * + * A leftover temporary from an interrupted store is still simply + * replaced -- that is what the unlink is for. The window between the + * two is not a hole: something recreated in it makes the create fail, + * which loses the store and keeps the target, rather than the other + * way round. + */ + if (nxt_slow_path(unlink(tmp_name) != 0 && nxt_errno != NXT_ENOENT)) { + nxt_alert(task, "unlink(\"%FN\") failed %E", file.name, nxt_errno); + + return NXT_ERROR; + } + + ret = nxt_file_open(task, &file, NXT_FILE_WRONLY, + NXT_FILE_CREATE_EXCLUSIVE, NXT_FILE_OWNER_ACCESS); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; } - n = nxt_file_write(&file, buf, size, 0); + /* + * pwrite() is allowed to store less than it was asked for, and this + * store exists to survive a partial one: resume from where it stopped + * rather than turning a short write into a failed store. A zero return + * carries no errno and cannot make progress, so it ends the loop. + */ + for (written = 0; written < size; written += n) { + n = nxt_file_write(&file, buf + written, size - written, written); + + if (nxt_slow_path(n <= 0)) { + /* nxt_file_write() logs the errno; a zero return has none. */ + if (n == 0) { + nxt_alert(task, "write(\"%FN\") stored %uz of %uz bytes", + file.name, written, size); + } + + goto fail; + } + } + + if (nxt_slow_path(nxt_main_file_store_inherit(task, &file, name) + != NXT_OK)) + { + goto fail; + } + + /* + * rename() already orders the name change for readers; what this buys + * is the data reaching stable storage before the name points at it, so + * a crash cannot leave conf.json naming a file whose blocks were never + * written. + */ + if (nxt_slow_path(nxt_file_sync(task, &file) != NXT_OK)) { + goto fail; + } nxt_file_close(task, &file); + file.fd = NXT_FILE_INVALID; - if (nxt_slow_path(n != (ssize_t) size)) { + if (nxt_slow_path(nxt_file_rename(file.name, (nxt_file_name_t *) name) + != NXT_OK)) + { (void) nxt_file_delete(file.name); return NXT_ERROR; } - return nxt_file_rename(file.name, (nxt_file_name_t *) name); + /* The destination is in place; a failed flush only costs durability. */ + (void) nxt_file_dir_sync(task, (nxt_file_name_t *) dir); + + return NXT_OK; + +fail: + + if (file.fd != NXT_FILE_INVALID) { + nxt_file_close(task, &file); + } + + (void) nxt_file_delete(file.name); + + return NXT_ERROR; +} + + +/* + * Carry the destination's mode and ownership onto the temporary that is + * about to replace it. A destination that does not exist yet, or that is + * not a regular file, keeps the 0600 the temporary was created with. + */ +static nxt_int_t +nxt_main_file_store_inherit(nxt_task_t *task, nxt_file_t *tmp, + const char *name) +{ + nxt_file_info_t fi; + + /* + * lstat(), not nxt_file_info(), which stat()s a name and therefore + * follows a symbolic link. The temporary is created exclusively + * because --statedir may be writable by another user; that same user + * can point the destination at a file of their own, and a stat() here + * would copy its ownership and mode onto the replacement -- aim it at + * something 0666 and conf.json comes back world-writable, and stays + * that way, because every later store inherits it again. + * + * On the path this is for -- an administrator's re-permissioned state + * file -- lstat() and stat() agree. + */ + if (lstat(name, &fi) != 0) { + if (nxt_errno == NXT_ENOENT) { + return NXT_OK; + } + + /* + * The temporary is written and 0600 is a serviceable result, so + * an unreadable destination costs the inheritance and not the + * store: refusing to persist the configuration at all is the worse + * outcome, which is the same trade the fchown() below makes. + */ + nxt_log(task, NXT_LOG_INFO, "lstat(\"%FN\") failed %E", + (nxt_file_name_t *) name, nxt_errno); + + return NXT_OK; + } + + /* + * Only a regular file donates anything. A symbolic link, or a + * destination somebody replaced with a device or a directory, leaves + * the temporary on the 0600 it was created with; the rename replaces + * whatever is there either way. + */ + if (!S_ISREG(fi.st_mode)) { + return NXT_OK; + } + + /* + * Ownership first, mode second. A successful chown() clears set-user-ID + * and set-group-ID on a regular file -- Linux does it to root's chown() + * too -- so doing it the other way round would drop exactly the bits + * "fi.st_mode & 07777" is here to carry over. + * + * Main runs as root, so this normally succeeds; when it cannot change + * the owner the store still proceeds -- refusing to persist the + * configuration would be the worse failure. + */ + if (nxt_slow_path(fchown(tmp->fd, fi.st_uid, fi.st_gid) != 0)) { + nxt_log(task, NXT_LOG_INFO, "fchown(%FD, \"%FN\") failed %E", + tmp->fd, tmp->name, nxt_errno); + } + + if (nxt_slow_path(fchmod(tmp->fd, fi.st_mode & 07777) != 0)) { + nxt_alert(task, "fchmod(%FD, \"%FN\") failed %E", + tmp->fd, tmp->name, nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; } diff --git a/src/nxt_main_process.h b/src/nxt_main_process.h index bb7fe68f1..7cb4cd351 100644 --- a/src/nxt_main_process.h +++ b/src/nxt_main_process.h @@ -37,6 +37,8 @@ extern const nxt_sig_event_t nxt_process_signals[]; void nxt_main_test_process_new_failures(nxt_uint_t failures); void nxt_main_test_run_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +nxt_int_t nxt_main_test_run_file_store(nxt_task_t *task, const char *dir, + const char *tmp_name, const char *name, u_char *buf, size_t size); #endif diff --git a/src/test/nxt_main_file_store_test.c b/src/test/nxt_main_file_store_test.c new file mode 100644 index 000000000..fb32dcfb2 --- /dev/null +++ b/src/test/nxt_main_file_store_test.c @@ -0,0 +1,421 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for issue #215: nxt_main_file_store() + * (src/nxt_main_process.c) ignored its "tmp_name" argument, opening the + * destination itself with NXT_FILE_TRUNCATE and finishing with a rename() + * of the file onto itself. The state file was rewritten in place, and the + * short-write path unlinked what was then the live conf.json. + * + * Four invariants, each observed rather than inferred: + * + * - a store replaces the destination by rename(2), pinned down by the + * inode number changing. An in-place rewrite keeps the inode, so this + * case alone separates the fixed function from the old one; + * - no temporary survives a successful store, including a stale one from + * an interrupted run left in the way; + * - a store that cannot create its temporary reports NXT_ERROR and leaves + * the destination byte-for-byte intact -- the shape of an ENOSPC or a + * read-only state directory, where the old code destroyed the + * configuration it was asked to update; + * - an existing destination keeps its mode across the replacement. + * + * Ownership is not asserted: the test does not require root, and chown() to + * another uid is not available to an unprivileged run. + */ + +#include +#include +#include +#include "nxt_tests.h" + +#include + + +#define nxt_main_file_store_test_fail(thr, ...) \ + do { \ + nxt_log_alert((thr)->log, "main file store test: " __VA_ARGS__); \ + goto done; \ + } while (0) + + +static nxt_int_t +nxt_main_file_store_test_content(const char *name, char *buf, size_t size, + ssize_t *len) +{ + int fd; + ssize_t n; + + fd = open(name, O_RDONLY); + if (nxt_slow_path(fd == -1)) { + return NXT_ERROR; + } + + n = read(fd, buf, size); + + close(fd); + + if (nxt_slow_path(n < 0)) { + return NXT_ERROR; + } + + *len = n; + + return NXT_OK; +} + + +static nxt_int_t +nxt_main_file_store_test_holds(const char *name, const char *expected) +{ + char buf[256]; + size_t len; + ssize_t n; + + if (nxt_slow_path(nxt_main_file_store_test_content(name, buf, sizeof(buf), + &n) + != NXT_OK)) + { + return NXT_ERROR; + } + + len = nxt_strlen(expected); + + if (nxt_slow_path((size_t) n != len || memcmp(buf, expected, len) != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_main_file_store_test_plant(const char *name, const char *content, + mode_t mode) +{ + int fd; + size_t len; + ssize_t n; + + fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, mode); + if (nxt_slow_path(fd == -1)) { + return NXT_ERROR; + } + + len = nxt_strlen(content); + + n = write(fd, content, len); + + close(fd); + + /* open() honours the umask; the test needs the exact mode. */ + if (nxt_slow_path(n != (ssize_t) len || chmod(name, mode) != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_main_file_store_test_store(nxt_task_t *task, const char *dir, + const char *tmp_name, const char *name, const char *content) +{ + return nxt_main_test_run_file_store(task, dir, tmp_name, name, + (u_char *) content, + nxt_strlen(content)); +} + + +nxt_int_t +nxt_main_file_store_test(nxt_thread_t *thr) +{ + char dir[NXT_MAX_PATH_LEN], name[NXT_MAX_PATH_LEN]; + char tmp_name[NXT_MAX_PATH_LEN], nowhere[NXT_MAX_PATH_LEN]; + char victim[NXT_MAX_PATH_LEN]; + ino_t first_ino; + const char *tmpdir; + nxt_int_t ret; + nxt_task_t *task; + struct stat st; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main file store test started"); + + task = thr->task; + task->thread = thr; + + tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || tmpdir[0] != '/') { + tmpdir = "/tmp"; + } + + (void) nxt_sprintf((u_char *) dir, (u_char *) dir + sizeof(dir), + "%s/nxt_main_file_store_test.XXXXXX%Z", tmpdir); + + if (nxt_slow_path(mkdtemp(dir) == NULL)) { + nxt_log_alert(thr->log, "main file store test: mkdtemp(\"%s\") " + "failed %E", dir, nxt_errno); + return NXT_ERROR; + } + + ret = NXT_ERROR; + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%s/conf.json%Z", dir); + (void) nxt_sprintf((u_char *) tmp_name, + (u_char *) tmp_name + sizeof(tmp_name), + "%s/conf.json.tmp%Z", dir); + (void) nxt_sprintf((u_char *) nowhere, + (u_char *) nowhere + sizeof(nowhere), + "%s/absent/conf.json.tmp%Z", dir); + (void) nxt_sprintf((u_char *) victim, + (u_char *) victim + sizeof(victim), + "%s/victim%Z", dir); + + /* A first store into an empty directory creates the destination. */ + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"one\":1}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the first store failed"); + } + + if (nxt_main_file_store_test_holds(name, "{\"one\":1}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" does not hold what the " + "first store wrote", name); + } + + if (stat(name, &st) != 0) { + nxt_main_file_store_test_fail(thr, "stat(\"%s\") failed %E", name, + nxt_errno); + } + + first_ino = st.st_ino; + + /* + * A stale temporary sits in the way, standing in for a store that was + * interrupted before its rename: it must be overwritten silently and + * must not survive. + */ + + if (nxt_main_file_store_test_plant(tmp_name, "stale", 0600) != NXT_OK) { + nxt_main_file_store_test_fail(thr, "could not plant a stale " + "temporary"); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"two\":2}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the second store failed"); + } + + if (nxt_main_file_store_test_holds(name, "{\"two\":2}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" does not hold what the " + "second store wrote", name); + } + + if (stat(tmp_name, &st) == 0) { + nxt_main_file_store_test_fail(thr, "\"%s\" was left behind by a " + "successful store", tmp_name); + } + + if (stat(name, &st) != 0) { + nxt_main_file_store_test_fail(thr, "stat(\"%s\") failed %E", name, + nxt_errno); + } + + if (st.st_ino == first_ino) { + nxt_main_file_store_test_fail(thr, "\"%s\" kept inode %uL across a " + "store -- the file was rewritten in " + "place, so a crash mid-write leaves " + "it truncated", name, + (uint64_t) first_ino); + } + + /* + * A store that cannot create its temporary -- the path names a + * directory that does not exist, the shape of a full or read-only state + * directory -- must report the failure and leave the destination as it + * was. + */ + + if (nxt_main_file_store_test_store(task, dir, nowhere, name, + "{\"three\":3}") == NXT_OK) + { + nxt_main_file_store_test_fail(thr, "a store whose temporary cannot " + "be created reported success"); + } + + if (nxt_main_file_store_test_holds(name, "{\"two\":2}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "a failed store damaged \"%s\"", + name); + } + + /* An existing destination keeps its mode across the replacement. */ + + if (chmod(name, 0640) != 0) { + nxt_main_file_store_test_fail(thr, "chmod(\"%s\") failed %E", name, + nxt_errno); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"four\":4}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the mode-preserving store " + "failed"); + } + + if (stat(name, &st) != 0 || (st.st_mode & 07777) != 0640) { + nxt_main_file_store_test_fail(thr, "\"%s\" came back with mode " + "0x%04xd, expected the 0640 (0x1a0) it had", + name, (unsigned) (st.st_mode & 07777)); + } + + /* + * Set-user-ID survives too. The replacement carries the mode over with + * fchmod() and the ownership with fchown(), and a chown() of a regular + * file clears set-user-ID -- so doing them in that order would drop the + * bit while appearing to preserve the mode. An unchanged owner is + * enough to trigger it, which is why this is assertable without root. + */ + + if (chmod(name, 04640) != 0) { + nxt_main_file_store_test_fail(thr, "chmod(\"%s\") failed %E", name, + nxt_errno); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"setid\":1}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the set-user-ID preserving store " + "failed"); + } + + if (stat(name, &st) != 0 || (st.st_mode & 07777) != 04640) { + nxt_main_file_store_test_fail(thr, "\"%s\" came back with mode " + "0x%04xd, expected the 04640 (0x9a0) " + "it had -- chown() after chmod() " + "clears set-user-ID", name, + (unsigned) (st.st_mode & 07777)); + } + + if (chmod(name, 0640) != 0) { + nxt_main_file_store_test_fail(thr, "chmod(\"%s\") failed %E", name, + nxt_errno); + } + + /* + * The temporary's name is predictable and the store runs as root, so a + * state directory writable by anyone else is a place to leave a symbolic + * link under that name and have the store follow it. Opening with + * O_TRUNC did: it truncated whatever the link addressed and the rename + * then installed the link as the state file. The store must write the + * state file and leave the link's target alone. + */ + + if (nxt_main_file_store_test_plant(victim, "precious", 0600) != NXT_OK) { + nxt_main_file_store_test_fail(thr, "could not plant the link target"); + } + + (void) unlink(tmp_name); + + if (symlink(victim, tmp_name) != 0) { + nxt_main_file_store_test_fail(thr, "symlink(\"%s\", \"%s\") failed " + "%E", victim, tmp_name, nxt_errno); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"five\":5}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "a store whose temporary was a " + "symbolic link failed"); + } + + if (nxt_main_file_store_test_holds(victim, "precious") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "the store followed the link and " + "wrote through it to \"%s\"", victim); + } + + if (nxt_main_file_store_test_holds(name, "{\"five\":5}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" does not hold what the " + "store past a link wrote", name); + } + + if (lstat(name, &st) != 0 || !S_ISREG(st.st_mode)) { + nxt_main_file_store_test_fail(thr, "\"%s\" is not a regular file -- " + "the link was installed as the state " + "file", name); + } + + (void) unlink(victim); + + /* + * The destination can be a symbolic link as well, and there the danger + * is not the write -- the rename replaces the link whatever it points + * at -- but the inheritance. Reading the destination's mode with + * stat() follows the link, so a link to a file of the other user's own + * hands them the mode of the conf.json that replaces it: aim it at + * something 0666 and every later store inherits that again. + */ + + if (nxt_main_file_store_test_plant(victim, "bait", 0666) != NXT_OK) { + nxt_main_file_store_test_fail(thr, "could not plant the mode bait"); + } + + (void) unlink(name); + + if (symlink(victim, name) != 0) { + nxt_main_file_store_test_fail(thr, "symlink(\"%s\", \"%s\") failed " + "%E", victim, name, nxt_errno); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "{\"six\":6}") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "a store onto a linked " + "destination failed"); + } + + if (nxt_main_file_store_test_holds(victim, "bait") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "the store wrote through the " + "linked destination to \"%s\"", victim); + } + + if (nxt_main_file_store_test_holds(name, "{\"six\":6}") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" does not hold what the " + "store onto a link wrote", name); + } + + if (lstat(name, &st) != 0 || !S_ISREG(st.st_mode)) { + nxt_main_file_store_test_fail(thr, "\"%s\" is not a regular file " + "after a store onto a link", name); + } + + if ((st.st_mode & 07777) != 0600) { + nxt_main_file_store_test_fail(thr, "\"%s\" came back with mode " + "0x%04xd, expected the 0600 (0x180) " + "the temporary was created with -- the " + "store inherited the link target's " + "mode", name, + (unsigned) (st.st_mode & 07777)); + } + + (void) unlink(victim); + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main file store test passed"); + + ret = NXT_OK; + +done: + + (void) unlink(name); + (void) unlink(tmp_name); + (void) unlink(victim); + (void) rmdir(dir); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 3821120e9..35b14e5ef 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -229,6 +229,10 @@ main(int argc, char **argv) return 1; } + if (nxt_main_file_store_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 53ef4fb89..cba76a724 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -81,6 +81,7 @@ nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_death_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); +nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); diff --git a/test/syscalls/baseline-linux-glibc.txt b/test/syscalls/baseline-linux-glibc.txt index bafcedfba..02bcdbb5f 100644 --- a/test/syscalls/baseline-linux-glibc.txt +++ b/test/syscalls/baseline-linux-glibc.txt @@ -21,8 +21,11 @@ epoll_wait eventfd2 execve exit_group +fchmod +fchown fcntl fstat +fsync ftruncate getdents64 getegid diff --git a/test/syscalls/baseline-linux-musl.txt b/test/syscalls/baseline-linux-musl.txt index 9571b15c3..df4716c79 100644 --- a/test/syscalls/baseline-linux-musl.txt +++ b/test/syscalls/baseline-linux-musl.txt @@ -19,9 +19,12 @@ epoll_pwait eventfd2 execve exit_group +fchmod +fchown fcntl fork fstat +fsync ftruncate futex getdents64 @@ -34,6 +37,7 @@ getsockopt gettid ioctl listen +lstat membarrier memfd_create mkdir diff --git a/test/syscalls/capture.sh b/test/syscalls/capture.sh index 1fbaed9d7..4d054e298 100755 --- a/test/syscalls/capture.sh +++ b/test/syscalls/capture.sh @@ -156,6 +156,23 @@ done grep -q 'syscall drift canary' "$WORK/body.txt" \ || die "served body is not the canary file" +# A second PUT, so the capture covers storing over state files that already +# exist. The first one writes conf.json and version into an empty statedir, +# where nxt_main_file_store_inherit() finds no destination and returns before +# it reads or copies anything -- so a single PUT never reaches the fchmod() +# and fchown() that carry a destination's mode and ownership onto its +# replacement, and the syscalls of the most privileged path in the store +# would be invisible to this gate. The settings block is here only to make +# the configuration differ from the one above; the listener and routes are +# untouched, so the canary served above still describes what is running. +sed -e 's/^{$/{\n "settings": {"http": {"header_read_timeout": 30}},/' \ + "$WORK/config.json" > "$WORK/config2.json" + +curl -sS --fail --max-time 10 --unix-socket "$SOCK" \ + -X PUT --data-binary "@$WORK/config2.json" \ + http://localhost/config >"$WORK/put2.log" 2>&1 \ + || { cat "$WORK/put2.log" >&2; die "the second configuration was rejected"; } + # Stop the daemon and let it run its teardown path *inside* the trace, so # shutdown syscalls are part of the captured set. Signal unitd, not strace # (see unit_pid above), then wait for strace itself to reap and flush. diff --git a/test/test_state_store.py b/test/test_state_store.py new file mode 100644 index 000000000..75f015be3 --- /dev/null +++ b/test/test_state_store.py @@ -0,0 +1,141 @@ +"""Persistence of the state directory (issue #215). + +The controller hands each accepted configuration to the main process, which +writes it to /conf.json. That write used to go straight into +conf.json with O_TRUNC, so a store that could not finish left the file +truncated -- and the short-write path unlinked it outright, which is what +this test observes on the unfixed code: a configuration too large for the +filesystem leaves no conf.json at all. + +The failure is arranged by putting the state directory on a 64 KiB tmpfs, +which needs root and CAP_SYS_ADMIN (pytest as root in a --privileged +container) as well as --restart; the test skips otherwise. +""" + +import json +import os +import shutil +import subprocess +import tempfile +import time +from pathlib import Path + +import pytest + +from conftest import unit_run, unit_stop +from unit.applications.proto import ApplicationProto +from unit.log import Log + +client = ApplicationProto() + + +SMALL_CONF = { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 200}}], +} + + +def big_conf(routes): + """Far more than the 64 KiB the filesystem below has.""" + + return { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"match": {"uri": f"/{'p' * 200}{i}"}, "action": {"return": 200}} + for i in range(routes) + ], + } + + +def wait_for_stored(statedir, expected, wait=100): + """The store is asynchronous: the controller answers the PUT and only + then asks the main process to persist it.""" + + conf_json = statedir / 'conf.json' + + for _ in range(wait): + try: + conf = json.loads(conf_json.read_text(encoding='utf-8')) + if conf.get('listeners') == expected['listeners']: + return conf + except (OSError, ValueError): + pass + + time.sleep(0.1) + + return None + + +def test_state_store_full_filesystem(skip_alert): + """A store that runs out of space must not damage the stored config.""" + + if os.geteuid() != 0: + pytest.skip('requires root to mount a tmpfs') + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + mounted = subprocess.run( + ['mount', '-t', 'tmpfs', '-o', 'size=64k', 'none', str(statedir)], + check=False, + capture_output=True, + ) + + if mounted.returncode != 0: + pytest.skip(f'could not mount a tmpfs: {mounted.stderr.decode()}') + + # The two alerts the failed store is expected to log. + skip_alert( + r'failed to store current configuration', + r'write\(.*conf\.json\.tmp', + ) + + try: + # main creates and writes everything here as root, so this only has + # to be traversable. 0777 would be the very condition the store is + # hardened against -- a state directory another user can plant a + # symbolic link in. + os.chmod(statedir, 0o755) + + unit_run(state_dir=str(statedir)) + + # The configuration that must survive. + assert 'success' in client.conf(SMALL_CONF), 'the small store' + assert wait_for_stored(statedir, SMALL_CONF) is not None, 'stored' + + stored = (statedir / 'conf.json').read_bytes() + + assert 'success' in client.conf(big_conf(400)), 'the large PUT' + + # The store is attempted asynchronously; wait for it to give up. + Log.wait_for_record(r'failed to store current configuration') + + # The store failed; the previously stored configuration is intact, + # byte for byte, and still parses. + after = (statedir / 'conf.json').read_bytes() + + assert after == stored, ( + 'conf.json was damaged by a store that could not complete' + ) + assert json.loads(after)['listeners'] == SMALL_CONF['listeners'] + + # And nothing was left half-written next to it. + assert [p.name for p in statedir.iterdir() if '.tmp' in p.name] == [] + + # "version" goes through the same function and is stored first, so a + # store that damaged it would leave the state directory unloadable + # even with conf.json intact. + version = statedir / 'version' + + assert version.is_file(), 'version survived the failed store' + assert version.read_bytes().strip() != b'', 'version is not empty' + + finally: + unit_stop() + subprocess.run( + ['umount', str(statedir)], check=False, capture_output=True + ) + # A failed umount would make rmdir() raise out of the finally and + # mask whichever assertion above actually failed. + shutil.rmtree(statedir, ignore_errors=True) From cc39416ca0d60ced27255556273553c0a4d7bdac Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 20:14:33 +0200 Subject: [PATCH 106/391] application: authenticate the prototype's START_PROCESS by its sender nxt_proto_start_process_handler() (src/nxt_application.c:623) checked nothing about who sent it. Its counterpart in main has verified the sender since #257 -- nxt_main_start_process_handler() refuses a START_PROCESS whose kernel-validated pid is not the router's (src/nxt_main_process.c:532) -- and the prototype, which forks the application workers themselves, had no such test. Every worker holds the write end of the port the message arrives on. nxt_proc_keep_matrix[] does not pair an APP with its prototype, but nxt_process_close_ports() keeps the parent's port unconditionally (src/nxt_process.c:372), and nxt_port_handler() dispatches on the wire type alone. So one compromised worker could make its own prototype fork more workers -- spending the process budget, and, once #268 has the prototype answer for a child that dies, driving the router's start bookkeeping from the inside. Only the router legitimately sends one (src/nxt_router.c:3347). The test has two arms, because the credential the kernel writes is relative to the receiver's pid namespace and the prototype has two views of the router: - Sharing main's namespace, the router is visible and the credential is its global pid, which is what the inherited rt->port_by_type[NXT_PROCESS_ROUTER] is keyed on. Same whitelist as main's. - Under "isolation": {"namespaces": {"pid": true}} the prototype is the init of its own namespace (src/nxt_process.c:765) and the router is in an ancestor of it, so the router has no pid there at all and the kernel delivers 0. Measured, not assumed: a socketpair across a CLONE_NEWPID boundary delivers SCM_CREDENTIALS pid 0 to the receiver, while a process inside the namespace delivers its non-zero namespace-local pid. Comparing against the router's global pid there would refuse every legitimate start -- the application would simply never come up -- so the isolated arm accepts a credential of 0 and nothing else. That is exactly the set the prototype does not fork: its workers get no clone flags of their own, so each is inside this namespace and presents a non-zero pid. The residual sender it admits is main, which is trusted and does not send START_PROCESS. A refused message is not answered. The reply addressing at "failed:" stays keyed on msg->port_msg.pid, because the runtime port hash is keyed on the global pid -- which in the isolated arm is the number the credential cannot supply -- and answering a forgery on a sender-chosen pid and port would let it cancel an RPC of its own choosing. Nothing is stranded by the silence: the router never armed an RPC for a message it did not send. The descriptors are closed, because this handler owns them and the port read loop reclaims nothing (src/nxt_port_socket.c:1381). The gate is compiled only where the platform hands the receiver a kernel-validated pid (NXT_USE_CMSG_PID), like every sibling gate. src/test/nxt_main_start_process_reply_test.c covers it: it already holds the main-side sender case, and this is the other end of the same message. Three refusals -- a foreign sender, no router port to compare against, and a worker inside a pid-isolated prototype -- each run in a forked child, because a handler that does not refuse goes on to fork from a fixture with no application loaded. On origin/master the first case reports exactly that: "did not refuse -- the handler ran past the sender check and into the start path, and died there on signal 11". test/test_php_isolation.py gains the black-box half: a PHP application under a pid namespace has to serve a request. The existing rootfs cases ask for that namespace only when the suite is not root, so as root -- how CI runs it -- nothing exercised it. Verified to have teeth: with the isolated arm deleted, so that the router's global pid is compared in both cases, the new test fails with "Can't read response from server". Unprivileged, unshare(CLONE_NEWPID) needs CAP_SYS_ADMIN, so running the suite as a normal user would fail the test before it reached the sender check at all. It therefore follows the shape test_go_isolation.py::test_isolation_pid already uses: the pid namespace is asked for unconditionally, and a non-root run additionally requires unprivileged user namespaces and enables the mount and credential ones, which is what supplies the capability. Co-Authored-By: Claude Opus 5 --- src/nxt_application.c | 117 ++++++++++ src/nxt_application.h | 9 + src/nxt_port.c | 5 +- src/test/nxt_main_start_process_reply_test.c | 225 +++++++++++++++++++ test/test_php_isolation.py | 43 ++++ 5 files changed, 397 insertions(+), 2 deletions(-) diff --git a/src/nxt_application.c b/src/nxt_application.c index 8d59663ce..ad739fa86 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -644,6 +644,90 @@ nxt_proto_start(nxt_task_t *task, nxt_process_data_t *data) } +#if (NXT_USE_CMSG_PID) + +/* + * Is the kernel-validated sender of this START_PROCESS allowed to make the + * prototype fork a worker? + * + * Only the router ever sends one (nxt_router_start_app_process_handler(), + * src/nxt_router.c:469, and nxt_router_app_prefork(), :3379), but every + * worker the prototype forks inherits the write end of the prototype's own + * port socket -- nxt_proc_keep_matrix[] does not list it, and + * nxt_process_close_ports() keeps the parent's port unconditionally -- and + * nxt_port_handler() dispatches on the wire type alone. So without this + * test one compromised worker can spend the prototype's process budget, and, + * once #268 has the prototype answer for a child that dies, drive the + * router's start bookkeeping from the inside. + * + * The test has two arms because the prototype has two pid views of the + * router, and the credential the kernel writes is always relative to the + * receiver's pid namespace: + * + * - Sharing main's namespace, the router is an ordinary visible process and + * the credential is its global pid, which is what the inherited + * rt->port_by_type[NXT_PROCESS_ROUTER] is keyed on. This is the same + * whitelist nxt_main_start_process_handler() applies + * (src/nxt_main_process.c:532). + * + * - Under "isolation": {"namespaces": {"pid": true}} the prototype is the + * init of its own namespace (nxt_process.c:765) and the router lives in + * an ancestor of it, so the router has no pid there at all: the kernel + * translates an untranslatable sender to 0 (pid_vnr() returns 0 outside + * the receiver's namespace; measured, not assumed). Comparing against + * the router's global pid would then refuse every legitimate start. A + * credential of 0 is therefore what "from outside this namespace" looks + * like, and it is exactly the set the prototype does not fork: its + * workers are forked with no clone flags of their own + * (nxt_proto_start_process_handler() leaves process->isolation zeroed), + * so each of them is inside this namespace and presents a non-zero + * namespace-local pid. What this arm authenticates is therefore a + * namespace boundary, not an identity: it admits every holder of the + * write end that sits outside this namespace. By construction that is + * main and the router (nxt_proc_keep_matrix[] keeps only those two in a + * worker, and nxt_proc_send_matrix[] lets the prototype answer only + * those two); main is trusted and does not send START_PROCESS. A worker + * that passes its inherited port out of the namespace over SCM_RIGHTS + * turns an accomplice into an admitted sender, which needs a second + * foothold and is the limit of what a credential can decide here. + * + * The reply addressing at "failed:" is deliberately left keyed on + * msg->port_msg.pid: the runtime port hash is keyed on the global pid, which + * in the isolated arm is precisely the number the credential cannot supply. + * That is why a refused message is not answered at all -- answering it would + * let the forger name any port and stream it likes and have the prototype + * cancel somebody else's RPC. A forgery strands nothing: the router never + * armed an RPC for a message it did not send. A genuine start refused by + * the router-port-is-NULL branch would be a different matter -- the router's + * RPC is keyed on the prototype (nxt_port_rpc_ex_set_peer(), src/nxt_router.c), + * so it would stay armed until the prototype dies -- but that branch is + * reachable only once the router is already gone, and a restarted router + * cannot address a pre-existing prototype at all: nxt_port_send_new_port() + * announces a newcomer outward and never announces existing peers to it. + */ +static nxt_bool_t +nxt_proto_start_process_sender_ok(nxt_task_t *task, nxt_runtime_t *rt, + nxt_port_recv_msg_t *msg) +{ + nxt_port_t *router_port; + + if (rt->is_pid_isolated) { + return nxt_recv_msg_cmsg_pid(msg) == 0; + } + + router_port = rt->port_by_type[NXT_PROCESS_ROUTER]; + + if (nxt_slow_path(router_port == NULL)) { + nxt_alert(task, "router port not found"); + return 0; + } + + return nxt_recv_msg_cmsg_pid(msg) == router_port->pid; +} + +#endif + + static void nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -656,6 +740,27 @@ nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; +#if (NXT_USE_CMSG_PID) + /* + * Before anything is allocated or forked, and without a reply: see + * nxt_proto_start_process_sender_ok(). The descriptors are closed here + * because this handler owns whatever the message carried -- the port + * read loop does not reclaim them (src/nxt_port_socket.c:1381). The + * router's own START_PROCESS to a prototype carries none (both fds are + * -1 at src/nxt_router.c:469 and :3379; the shared port and queue only + * travel in the one it sends main), so this is for what a forger + * attaches, not for anything a legitimate message brings. + */ + if (nxt_slow_path(!nxt_proto_start_process_sender_ok(task, rt, msg))) { + nxt_alert(task, "process %PI cannot start processes", + nxt_recv_msg_cmsg_pid(msg)); + + nxt_port_recv_msg_close_fds(msg); + + return; + } +#endif + process = nxt_process_new(rt); if (nxt_slow_path(process == NULL)) { goto failed; @@ -721,6 +826,18 @@ nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +#if (NXT_TESTS) + +void +nxt_proto_test_run_start_process_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg) +{ + nxt_proto_start_process_handler(task, msg); +} + +#endif + + static void nxt_proto_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { diff --git a/src/nxt_application.h b/src/nxt_application.h index a3b4230a0..cb547fa96 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -167,6 +167,15 @@ struct nxt_app_module_s { nxt_app_lang_module_t *nxt_app_lang_module(nxt_runtime_t *rt, nxt_str_t *name); nxt_app_type_t nxt_app_parse_type(u_char *p, size_t length); +#if (NXT_TESTS) +/* + * Invokes the static nxt_proto_start_process_handler() with a synthesised + * runtime and message; see src/test/nxt_main_start_process_reply_test.c. + */ +void nxt_proto_test_run_start_process_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +#endif + NXT_EXPORT extern nxt_str_t nxt_server; extern nxt_app_module_t nxt_external_module; diff --git a/src/nxt_port.c b/src/nxt_port.c index e5f184c23..e39d7784b 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -307,8 +307,9 @@ nxt_port_enable(nxt_task_t *task, nxt_port_t *port, * nxt_port_socket.c fill msg->cmsg_pid. Individual handlers are * being gated on nxt_recv_msg_cmsg_pid() one at a time -- * nxt_main_process_created_handler(), nxt_main_start_process_handler(), - * nxt_port_process_ready_handler(), nxt_proto_process_created_handler() - * and the cert/script/socket/access-log handlers already are. + * nxt_port_process_ready_handler(), nxt_proto_process_created_handler(), + * nxt_proto_start_process_handler() and the cert/script/socket/ + * access-log handlers already are. * * What is left is the table-driven part: a per-message-type sender * ACL applied here, in the dispatcher, so that a handler which diff --git a/src/test/nxt_main_start_process_reply_test.c b/src/test/nxt_main_start_process_reply_test.c index 6432b9f21..7f610a7b0 100644 --- a/src/test/nxt_main_start_process_reply_test.c +++ b/src/test/nxt_main_start_process_reply_test.c @@ -32,6 +32,17 @@ * ever cancel a stream of the forger's own; that case gives the two pids * different values and requires the router's port to stay untouched. * + * The prototype's own START_PROCESS handler is covered here too. It is the + * other end of the same message -- the router sends one to main to get a + * prototype and one to the prototype to get a worker + * (src/nxt_router.c:469 and :3379) -- and until issue #270 it checked + * nothing at all, although every worker it forks inherits the write end of + * the prototype's port. Its cases are refusals only, and each is run in a + * forked child: a handler that does not refuse goes on to fork a worker + * from a fixture that has no application loaded, and containing that keeps + * one broken gate from taking the whole test binary down. The parent + * reports a child that died as the fall-through it is. + * * Not covered: the nxt_mp_create() failure a few lines below the process * allocation. Arming it needs a malloc-failure hook that does not exist, * and the sites are not independent -- both reach "failed:" through the @@ -45,10 +56,12 @@ #include #include #include +#include #include #include "nxt_tests.h" #include +#include static nxt_bool_t @@ -268,6 +281,130 @@ nxt_main_start_process_reply_test_no_reply_case(nxt_thread_t *thr, return ret; } +#if (NXT_USE_CMSG_PID) + +/* + * One refusal case for nxt_proto_start_process_handler(), run inside a + * forked child. + * + * The child does the whole check and reports through its exit status: 0 when + * the handler refused, 1 when it was reached and left a trace it should not + * have. A handler with no gate does not return at all -- it allocates a + * process and reads nxt_app_conf, which no fixture here sets -- so the + * parent treats a signalled child as the failure it is, and names it. + * + * What "refused" means, in the order the child tests it: nothing was queued + * on any port (a refusal must not answer, because the reply is addressed by + * the sender-chosen msg->port_msg.pid and answering would let a forger + * cancel a stream of its choosing), and both descriptors the message carried + * were closed (this handler owns them; the port read loop reclaims nothing). + */ +static nxt_int_t +nxt_main_start_process_reply_test_proto_case(nxt_thread_t *thr, + nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_t *port_a, + nxt_port_t *port_b, const char *name) +{ + int status; + pid_t child; + nxt_fd_t fd0, fd1; + nxt_int_t ret; + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/null", O_RDONLY); + + if (nxt_slow_path(fd0 == -1 || fd1 == -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s failed to " + "open the descriptors", name); + + if (fd0 != -1) { + (void) close(fd0); + } + + if (fd1 != -1) { + (void) close(fd1); + } + + return NXT_ERROR; + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + + child = fork(); + + if (nxt_slow_path(child == -1)) { + nxt_log_alert(thr->log, "main start process reply test: %s failed to " + "fork %E", name, nxt_errno); + (void) close(fd0); + (void) close(fd1); + + return NXT_ERROR; + } + + if (child == 0) { + ret = NXT_OK; + + nxt_proto_test_run_start_process_handler(task, msg); + + if (!nxt_queue_is_empty(&port_a->messages) + || (port_b != NULL && !nxt_queue_is_empty(&port_b->messages))) + { + nxt_log_alert(thr->log, "main start process reply test: %s was " + "answered; a refused START_PROCESS must not reply, " + "because the reply is addressed by the pid the " + "sender chose", name); + ret = NXT_ERROR; + } + + if (msg->fd[0] != -1 || msg->fd[1] != -1 + || nxt_main_start_process_reply_test_fd_is_open(fd0) + || nxt_main_start_process_reply_test_fd_is_open(fd1)) + { + nxt_log_alert(thr->log, "main start process reply test: %s leaked " + "a descriptor", name); + ret = NXT_ERROR; + } + + /* + * _exit(), not exit(): the child shares the parent's stdio and its + * pools, and must run no destructor or atexit handler of theirs. + */ + _exit(ret == NXT_OK ? 0 : 1); + } + + /* The child owns them now; this frame only has to stop naming them. */ + (void) close(fd0); + (void) close(fd1); + + msg->fd[0] = -1; + msg->fd[1] = -1; + + while (waitpid(child, &status, 0) == -1) { + if (nxt_errno != NXT_EINTR) { + nxt_log_alert(thr->log, "main start process reply test: %s failed " + "to reap the child %E", name, nxt_errno); + return NXT_ERROR; + } + } + + if (nxt_slow_path(!WIFEXITED(status))) { + nxt_log_alert(thr->log, "main start process reply test: %s did not " + "refuse -- the handler ran past the sender check and " + "into the start path, and died there on signal %d", + name, WTERMSIG(status)); + return NXT_ERROR; + } + + if (nxt_slow_path(WEXITSTATUS(status) != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + +#endif + + nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr) { @@ -474,6 +611,94 @@ nxt_main_start_process_reply_test(nxt_thread_t *thr) goto done; } +#if (NXT_USE_CMSG_PID) + + /* + * The prototype's START_PROCESS handler, issue #270. Each case is a + * refusal; the fixture is the one above, read as the prototype's own + * runtime -- rt->port_by_type[NXT_PROCESS_ROUTER] is the router port the + * prototype inherited from main, which nxt_proc_keep_matrix[] keeps. + */ + + /* + * A worker forging a START_PROCESS. It claims to be the router in the + * one field it controls, and the credential names it. This is the whole + * exposure: nxt_process_close_ports() keeps the parent's port + * unconditionally, so every worker holds the write end of the port this + * message arrived on. + */ + + msg.cmsg_pid = nxt_pid + 1; + msg.port_msg.stream = 0x55555555; + + ret = nxt_main_start_process_reply_test_proto_case(thr, task, &msg, + router_port, + foreign_port, + "a foreign sender to " + "the prototype"); + + msg.cmsg_pid = nxt_pid; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * No router port to compare against. The prototype cannot tell who sent + * this, and refusing is the only safe answer -- the same conclusion + * nxt_main_start_process_handler() reaches, minus the reply it can + * address and the prototype cannot. + */ + + rt->port_by_type[NXT_PROCESS_ROUTER] = NULL; + + msg.port_msg.stream = 0x66666666; + + ret = nxt_main_start_process_reply_test_proto_case(thr, task, &msg, + router_port, + foreign_port, + "a missing router port " + "in the prototype"); + + rt->port_by_type[NXT_PROCESS_ROUTER] = router_port; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A pid-isolated prototype. There the router is in an ancestor pid + * namespace and has no pid at all in this one, so the kernel writes a + * credential of 0 and the router's global pid is the wrong thing to + * compare against; the prototype's own workers, forked with no clone + * flags of their own, are inside the namespace and carry non-zero + * namespace-local pids. This case is such a worker -- a credential that + * is neither 0 nor the router's -- and pins the arm down from the side + * that matters: a gate that simply skipped the check when isolated would + * pass every other case here and fail this one. + */ + + rt->is_pid_isolated = 1; + + msg.cmsg_pid = 2; + msg.port_msg.stream = 0x77777777; + + ret = nxt_main_start_process_reply_test_proto_case(thr, task, &msg, + router_port, + foreign_port, + "a worker inside a " + "pid-isolated " + "prototype"); + + rt->is_pid_isolated = 0; + msg.cmsg_pid = nxt_pid; + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + +#endif + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "main start process reply test passed"); diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index 918975c00..e88fb0da3 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -211,3 +211,46 @@ def test_php_isolation_rootfs_credential_no_automount(is_su, require, ) assert client.get()['status'] == 200, 'no automount serves' + + +def test_php_isolation_pid_namespace(is_su, require): + """A prototype in its own pid namespace still starts workers. + + The router's START_PROCESS reaches such a prototype from an ancestor pid + namespace, so the credential the kernel attaches to it is 0 rather than + the router's pid: the prototype has no pid for the router at all. The + sender check in nxt_proto_start_process_handler() (issue #270) has to + know that, and a version of it that simply compared against the router's + global pid would refuse every start here -- with the application never + coming up, which is what this asks about. + + Unlike the rootfs cases above, the pid namespace is requested whether or + not the suite runs as root: as root is exactly where the other tests do + not ask for one. Unprivileged, unshare(CLONE_NEWPID) needs CAP_SYS_ADMIN, + so a user namespace has to come with it -- the same shape as + test_go_isolation.py::test_isolation_pid. + """ + require({'features': {'isolation': ['pid']}}) + + if not is_su: + require( + { + 'features': { + 'isolation': [ + 'unprivileged_userns_clone', + 'user', + 'mnt', + ] + } + } + ) + + isolation = {'namespaces': {'pid': True}} + + if not is_su: + isolation['namespaces']['mount'] = True + isolation['namespaces']['credential'] = True + + client.load('phpinfo', isolation=isolation) + + assert client.get()['status'] == 200, 'pid-isolated prototype' From f0fb2eb152e32769d985bc1f7115e6149269dba1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 20:15:05 +0200 Subject: [PATCH 107/391] process: retire the start stream when its reply goes out, not on READY process->stream is the RPC the initiator armed for a start. While it is set, nxt_port_remove_notify_others() puts it into the REMOVE_PID that reports the process's death (src/nxt_port.c:1031), and nxt_router_remove_pid_handler() turns a stream-bearing REMOVE_PID into an RPC_ERROR (src/nxt_router.c:1128-1133). A worker of the prototype kept ->stream set for its whole life, so every routine worker exit asked the router to fail whatever RPC held that number. Harmless while the number is dead -- and not permanently dead: stream identifiers come from one 32-bit counter that every request also draws on (nxt_stream_ident, src/nxt_port_rpc.c:11 and :164; the per-request draw is src/nxt_router.c:5880), so after a wrap an ordinary worker exit retires an unrelated live RPC. Main did not have that, but it avoided it in the wrong place. nxt_main_process_sigchld_handler() cleared ->stream for a child that had reached READY. Copying that to the prototype was drafted and dropped in review, because the READY state is not the moment the start RPC is answered: nxt_port_process_ready_handler() sets the state and only then calls nxt_port_send_new_port(), ignoring its return. In that window nothing has retired the RPC, and ->stream in a later REMOVE_PID is the only thing that still can -- so clearing on the state alone drops the fallback for a start whose reply never went out, which is the wedge issue #231 removes, re-entered through a corner. Key it on the reply instead. nxt_port_send_new_port() now reports whether the announcement that answers the start was accepted for delivery, and the handler clears ->stream only then. That announcement is the one addressed to the router, and only that one. The router owns the registration for every start: it registers the RPC on its own port and then sends START_PROCESS elsewhere -- to main when a prototype has to be forked and to the prototype when a worker has, but the handler is armed on the router port in both cases (nxt_router_start_app_process_handler() registers on the router port it is called with, src/nxt_router.c:5001 and :459; nxt_router_app_prefork() on rt->port_by_type[NXT_PROCESS_ROUTER], src/nxt_router.c:3347). Main forks the prototype but never owns the RPC for it. nxt_router_new_port_handler() then feeds a stream-bearing NEW_PORT to nxt_port_rpc_handler() (src/nxt_router.c:768-771), which retires the registration. Aggregating every peer instead would be wrong in the direction that costs correctness: when the prototype announces a worker it writes to main as well as to the router, and a failure to main with the router's write accepted would keep a stream the router has already retired -- exactly the wrapped-counter hazard this clears it to avoid. A failed write to any other peer is logged and does not change the answer. With no router peer at all NXT_OK is the honest answer: that is the boot-time shape, where main announces the core processes with stream 0 and clearing is a no-op. The stream announced is process->stream, the value this process itself registered when it forked the child, not the one the message carries. A child echoes the stream it inherited across the fork (nxt_process_send_ready(), src/nxt_process.c:1142), so for a healthy child the two are the same number -- but the wire field is written by the sender, and the gate added in 4c79840d authenticates who sent a PROCESS_READY, not what it says. Taking the echoed value would let a child that named a different stream, or zero, have the announcement carry a number no registration is waiting on while this cleared the one that is: the router would never retire the real start, and the REMOVE_PID that is the remaining fallback would carry nothing -- a start pending forever, which is worse than what this fixes. A mismatch is logged rather than refused, because refusing would strand the start the receiver does know about. This is in the handler both main and the prototype install, so main gets the same treatment and its sigchld clearing is removed rather than kept alongside: the new one is strictly earlier and strictly narrower. Two comments that cited the old site are rewritten. src/test/nxt_port_ready_test.c covers it, with a process announcing itself with no queue descriptor on a prototype port -- a real shape whose announcement carries no descriptors -- and two peers, a router and a main, since nxt_proc_send_matrix[] pairs both with PROTOTYPE and one peer could not tell the two rules apart. Five cases: with every peer's allocation failed the stream survives; with none failed it is cleared and the announcement is on the router's port carrying that stream; a READY naming a foreign stream and one naming stream 0 must still announce the registered stream and retire it; and with one allocation failed, the stream is cleared exactly when the router was the peer that got through. Each was shown to fail without the code it tests. Against origin/master the healthy case reports "the start stream survived an announcement that answered it". Announcing the echoed stream fails the foreign-stream case with "announced the port on stream #3735928559, but the start it answers is stream #555819297". Aggregating every peer fails the mixed case with "the router was announced to and retired the start, but the stream was kept because another peer's write failed". Retiring the stream leaves the router a case it did not have before: a worker that sends PROCESS_READY a second time is announced again, now with no stream, and an application's main port has id 0. That pair reached the "new application port" arm, whose nxt_assert(port->id != 0) aborts a debug build and, compiled out of a release build, let the port be re-added to the application hash -- nxt_port_hash_add() declines the duplicate key, but ->port_hash_count is incremented regardless, and nxt_router_app_need_start() reads that count. nxt_router_new_port_handler() now recognises a port it already holds and returns -- and, for a main port it has never seen announced with no stream at all, undoes the registration this message caused rather than leaving an orphan in the runtime. Telling the two apart needs to know whether nxt_port_new_port_handler() found the port or created it, which it now reports in msg->new_port_created. nxt_router_new_port_test.c covers both. Co-Authored-By: Claude Opus 5 --- src/nxt_main_process.c | 12 +- src/nxt_port.c | 153 ++++++++- src/nxt_port.h | 8 + src/nxt_process.c | 11 +- src/nxt_router.c | 49 ++- src/test/nxt_port_ready_test.c | 467 ++++++++++++++++++++++++++++ src/test/nxt_router_new_port_test.c | 121 ++++++- 7 files changed, 798 insertions(+), 23 deletions(-) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index c5c5b59bf..701215d24 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -1213,9 +1213,15 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) if (process != NULL) { nxt_main_process_cleanup(task, process); - if (process->state == NXT_PROCESS_STATE_READY) { - process->stream = 0; - } + /* + * ->stream is no longer cleared here. It is cleared where the + * start RPC is actually answered -- on a successful NEW_PORT + * announcement in nxt_port_process_ready_handler() -- which is + * both earlier and narrower: the READY state is set before that + * announcement is written, so clearing on the state dropped the + * REMOVE_PID fallback for a start whose reply never went out. + * See issue #271. + */ nxt_queue_init(&children); diff --git a/src/nxt_port.c b/src/nxt_port.c index e39d7784b..940225785 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -367,11 +367,54 @@ nxt_port_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Announce a port to every peer the send matrix pairs it with, carrying + * "stream" so that the announcement doubles as the reply to the start RPC + * the initiator armed. + * + * Returns NXT_OK when the announcement that answers that RPC was accepted + * for delivery, so that the caller can read it as "the start has been + * answered". + * + * That is the announcement addressed to the router, and only that one. The + * router owns the registration for every start: it registers the RPC on its + * own port and then sends START_PROCESS elsewhere -- to main when a + * prototype has to be forked and to the prototype when a worker has, but the + * handler is armed on the router port in both cases + * (nxt_router_start_app_process_handler() registers on the router port it + * was called with, src/nxt_router.c:5069 and :459; + * nxt_router_app_prefork() on rt->port_by_type[NXT_PROCESS_ROUTER], + * src/nxt_router.c:3365). Main forks the prototype but never owns the RPC + * for it. nxt_router_new_port_handler() then feeds a stream-bearing + * NEW_PORT to nxt_port_rpc_handler() (src/nxt_router.c:788-791), which is + * what retires the registration -- when it gets that far. NXT_OK here means + * the announcement was queued for the router, not that the router acted on + * it: nxt_router_new_port_handler() returns at src/nxt_router.c:782 when it + * cannot map the new port's queue, walking past the RPC dispatch and leaving + * the start outstanding. Retiring the stream on a queued write is therefore + * the best answer available here and not a guarantee; issue #223 is where + * the router learns to fail the start it refused. + * + * Aggregating every peer instead would be wrong in the direction that costs + * correctness: when the prototype announces a worker it writes to main as + * well as to the router, and a failure to main with the router's write + * accepted would keep a stream the router has already retired -- exactly the + * wrapped-counter hazard the caller clears it to avoid. A failed write to + * any other peer is logged and does not change the answer; it is a delivery + * problem for that peer, not evidence about the start. + * + * With no router peer at all there is nothing to report on and NXT_OK is the + * honest answer: an initiator that is not in this runtime has no + * registration here to keep alive for. That is the boot-time shape, where + * main announces the core processes with stream 0 and clearing is a no-op. + */ + /* TODO join with process_ready and move to nxt_main_process.c */ -nxt_inline void +nxt_inline nxt_int_t nxt_port_send_new_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_port_t *new_port, uint32_t stream) { + nxt_int_t ret; nxt_port_t *port; nxt_process_t *process; @@ -382,19 +425,36 @@ nxt_port_send_new_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_debug(task, "new port %d for process %PI", new_port->pair[1], new_port->pid); + ret = NXT_OK; + nxt_runtime_process_each(rt, process) { - if (process->pid == new_port->pid || process->pid == nxt_pid) { + if (process->pid == new_port->pid || process->pid == nxt_pid + || nxt_queue_is_empty(&process->ports)) + { continue; } port = nxt_process_port_first(process); if (nxt_proc_send_matrix[port->type][new_port->type]) { - (void) nxt_port_send_port(task, port, new_port, stream); + if (nxt_slow_path(nxt_port_send_port(task, port, new_port, stream) + != NXT_OK)) + { + if (port->type == NXT_PROCESS_ROUTER) { + ret = NXT_ERROR; + + } else { + nxt_log(task, NXT_LOG_WARN, "failed to announce the port " + "of process %PI to process %PI", new_port->pid, + port->pid); + } + } } } nxt_runtime_process_loop; + + return ret; } @@ -445,6 +505,7 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) * so rather than leave the garbage the stack happened to hold. */ msg->u.new_port = NULL; + msg->new_port_created = 0; new_port_msg = (nxt_port_msg_new_port_t *) msg->buf->mem.pos; @@ -517,6 +578,7 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_write_enable(task, port); msg->u.new_port = port; + msg->new_port_created = 1; /* * fd[1] is deliberately left in the message: it is the queue of the @@ -533,6 +595,7 @@ void nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { void *mem; + uint32_t stream; nxt_port_t *port; nxt_process_t *process; nxt_runtime_t *rt; @@ -616,9 +679,11 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) * Every guard here rejects before the state is set: a message that is * not acted upon must leave no trace. Marking a process ready and only * then rejecting it would make the next, legitimate PROCESS_READY trip - * the guard above, and would let nxt_main_process_sigchld_handler() - * clear the start stream of a process that never finished starting, - * dropping the REMOVE_PID that cancels the pending start RPC. + * the guard above -- and, before the start stream was tied to the + * announcement at the tail of this handler, would have let + * nxt_main_process_sigchld_handler() clear the start stream of a process + * that never finished starting, dropping the REMOVE_PID that cancels the + * pending start RPC. */ if (nxt_slow_path(nxt_queue_is_empty(&process->ports))) { nxt_log(task, NXT_LOG_WARN, "PROCESS_READY claiming process %PI, " @@ -764,10 +829,11 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) * kill the worker and still leave the start pending, which * is the wedge this arm exists to end. * - * CREATED, not READY: it never became usable, and on the - * paths main reports rather than the prototype, READY is - * what makes main clear the start stream - * (nxt_main_process.c:1108-1110). + * CREATED, not READY: it never became usable. The start + * stream survives either way -- this arm returns above the + * announcement at the tail of this handler, which is the + * only thing that clears it -- so the REMOVE_PID this + * guarantees still carries it. */ if (process->state == NXT_PROCESS_STATE_CREATING) { process->state = NXT_PROCESS_STATE_CREATED; @@ -840,7 +906,72 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) */ nxt_port_recv_msg_close_fds(msg); - nxt_port_send_new_port(task, rt, port, msg->port_msg.stream); + /* + * Retire the start stream, but only once the announcement that answers + * it has actually gone out. + * + * ->stream is the RPC the initiator armed for this start. While it is + * set, nxt_port_remove_notify_others() puts it into the REMOVE_PID that + * reports this process's death, and nxt_router_remove_pid_handler() + * turns a stream-bearing REMOVE_PID into an RPC_ERROR + * (src/nxt_router.c:1147-1153). That is the right fallback for a start + * that never completed, and a liability afterwards: stream identifiers + * come from one 32-bit counter (nxt_stream_ident, src/nxt_port_rpc.c:11, + * bumped at src/nxt_port_rpc.c:164) that every request also draws on + * (src/nxt_router.c:5880), so once it wraps, an ordinary worker exit + * would fail whatever live RPC has inherited the number. The reachable + * collision set is small -- the retype lands on the router's main port, + * which holds start, prefork, listen-socket and access-log + * registrations, while request RPCs live on the worker threads' engine + * ports -- but it is not empty. + * + * Zeroing on the READY state alone is what this deliberately is not. + * The state is set above and the announcement is sent here, and in + * between the start RPC has not been retired by anything -- so a + * PROCESS_READY whose NEW_PORT could not be written would lose both the + * reply and the REMOVE_PID fallback, and leave the initiator waiting + * forever. That is the wedge issue #231 is about, re-entered through a + * corner. Keyed on the send instead, the stream survives exactly the + * cases that still need it. See issue #271. + * + * nxt_main_process_sigchld_handler() used to do this, later and on the + * state; it no longer needs to, and main gets the same treatment here + * because it runs the same handler for its own children. + * + * The stream announced is process->stream, the value this process itself + * registered when it forked the child (nxt_main_start_process_handler() + * and nxt_proto_start_process_handler()), not the one the message + * carries. A child echoes the stream it inherited across the fork + * (nxt_process_send_ready(), src/nxt_process.c:1203), so for a healthy + * child the two are the same number -- but the wire field is written by + * the sender, and the sender gate above authenticates who sent the + * message, not what it says. A child that named a different stream, or + * zero, would otherwise have the announcement carry a number no + * registration is waiting on while this cleared the one that is: the + * router would never retire the real start, and the REMOVE_PID that is + * the remaining fallback would carry nothing. Announcing the registered + * value and clearing that same value keeps the reply and the fallback + * describing one start. + * + * A mismatch is logged rather than refused. Refusing would strand the + * start the receiver knows about, which is the failure this is avoiding; + * announcing the authoritative stream answers it correctly and leaves a + * record that a child sent something it should not have. + */ + stream = process->stream; + + if (nxt_slow_path(msg->port_msg.stream != stream)) { + nxt_log(task, NXT_LOG_WARN, "process %PI sent PROCESS_READY naming " + "start stream #%uD, but it was started for stream #%uD; " + "answering the latter", msg->port_msg.pid, + msg->port_msg.stream, stream); + } + + if (nxt_fast_path(nxt_port_send_new_port(task, rt, port, stream) + == NXT_OK)) + { + process->stream = 0; + } } diff --git a/src/nxt_port.h b/src/nxt_port.h index 83defcfab..79b05d42f 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -227,6 +227,14 @@ struct nxt_port_recv_msg_s { nxt_pid_t cmsg_pid; #endif nxt_bool_t cancelled; + /* + * Set by nxt_port_new_port_handler() when it had to create the port + * u.new_port names, clear when it found one already registered. A + * caller that refuses the announcement needs the difference: an existing + * port is live and must be left alone, while one this message brought + * into the runtime is the caller's to undo. + */ + nxt_bool_t new_port_created; union { nxt_port_t *new_port; nxt_pid_t removed_pid; diff --git a/src/nxt_process.c b/src/nxt_process.c index 2680f39e3..42a797b8e 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -1010,11 +1010,12 @@ nxt_process_created_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) * * The refusal is visible: this exits nonzero, main's SIGCHLD * reaper notifies the router with the start's stream still - * attached (main zeroes ->stream only at state READY, and its - * copy of a prototype that died here is still CREATED), the - * router turns that REMOVE_PID into an RPC error for the start - * attempt, and the requests waiting on the application are - * answered 503 rather than left to time out. + * attached (main clears ->stream only once the NEW_PORT that + * answers the start has gone out, and a prototype that died + * here never sent the PROCESS_READY that would have caused + * one), the router turns that REMOVE_PID into an RPC error for + * the start attempt, and the requests waiting on the application + * are answered 503 rather than left to time out. */ nxt_alert(task, "%s refused to start: capset() is denied, so the " diff --git a/src/nxt_router.c b/src/nxt_router.c index 1c8395414..ce1d09564 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -793,10 +793,53 @@ nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "new port id %d (%d)", port->id, port->type); /* - * Port with "id == 0" is application 'main' port and it always - * should come with non-zero stream. + * An application's "main" port has id 0 and is announced carrying the + * stream of the start it answers, which the dispatch above consumes. + * Arriving here means id 0 with no stream, and the arm below cannot + * serve that pair: it looks the announcement up as a *sibling* of an + * already-known main port, and for id 0 that lookup finds the port + * itself. Which of the two cases this is decides the remedy, and only + * nxt_port_new_port_handler() knows -- hence msg->new_port_created. + * + * Already registered: a worker that sent PROCESS_READY a second time. + * Its start stream was retired by the first announcement + * (nxt_port_process_ready_handler(), src/nxt_port.c), so the repeat + * carries none. The port is live and its queue, if it brought one, was + * mapped above -- leave it alone. Falling through would re-add it to + * the application hash: nxt_port_hash_add() declines the duplicate key, + * but ->port_hash_count is incremented regardless, and + * nxt_router_app_need_start() reads that count, so the router would + * believe it has workers it does not have. It would also send a second + * PORT_ACK for one port. + * + * Created by this message: an application main port announced with no + * stream at all, which no start is waiting on and nothing else will + * complete. Keeping it would leave a port registered in the runtime + * that no application owns and no PORT_ACK was ever sent for. Undo the + * registration this message caused instead. + * + * An assertion is not enough for either: nxt_assert() compiles out in a + * release build, which is exactly where the miscount and the orphan + * would do their damage. */ - nxt_assert(port->id != 0); + if (nxt_slow_path(port->id == 0)) { + + if (msg->new_port_created) { + nxt_alert(task, "new port of process %PI has id 0 and no start " + "stream; refused", port->pid); + + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + + return; + } + + nxt_log(task, NXT_LOG_WARN, "process %PI announced its main port " + "again with no start stream; already registered, ignored", + port->pid); + + return; + } /* Find 'main' app port and get app reference. */ rt = task->thread->runtime; diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index f30801d58..b86bc10e6 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -25,12 +25,17 @@ * pid-isolated process -- pid and isolated_pid deliberately differ -- so * that the cases below can tell the two apart: a credential equal to the * global pid has to be refused just like a foreign one. + * + * The last case covers what the handler does with process->stream, the start + * RPC the initiator armed: it is retired when the announcement that answers + * that RPC has gone out, and kept when it has not. See issue #271. */ #include #include #include #include +#include #include "nxt_tests.h" #include @@ -867,6 +872,459 @@ nxt_port_ready_test_queueless(nxt_thread_t *thr, nxt_task_t *task, #endif +/* + * Require that the announcement the handler just made is on the router's + * port and carries "expect" as its stream, then drain both peer ports so the + * next case starts from empty queues. + * + * The stream is what these cases are about: it is the number the router + * matches against its registered start RPCs, so announcing the wrong one + * answers nobody while looking, from every other angle, like a success. + * + * "expect" may be NXT_PORT_READY_TEST_NO_MSG to require that the router was + * not announced to at all, or NXT_PORT_READY_TEST_ANY to make no + * requirement and only report, through *got, whether it was -- which is what + * the mixed-outcome case below needs, since which of the two peers a single + * armed allocation failure lands on is hash order, not something this test + * fixes. + */ + +#define NXT_PORT_READY_TEST_NO_MSG ((uint32_t) -1) +#define NXT_PORT_READY_TEST_ANY ((uint32_t) -2) + +static nxt_int_t +nxt_port_ready_test_announced(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *router_port, nxt_port_t *other_port, uint32_t expect, + nxt_bool_t *got, const char *name) +{ + nxt_int_t ret; + nxt_uint_t n; + nxt_queue_link_t *link; + nxt_port_send_msg_t *sent; + + ret = NXT_OK; + sent = NULL; + n = 0; + + for (link = nxt_queue_first(&router_port->messages); + link != nxt_queue_tail(&router_port->messages); + link = nxt_queue_next(link)) + { + n++; + sent = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + } + + if (got != NULL) { + *got = (n != 0); + } + + if (expect == NXT_PORT_READY_TEST_ANY) { + /* Reported, not required. */ + + } else if (expect == NXT_PORT_READY_TEST_NO_MSG) { + if (nxt_slow_path(n != 0)) { + nxt_log_alert(thr->log, "port ready test: %s announced the port " + "to the router anyway", name); + ret = NXT_ERROR; + } + + } else if (nxt_slow_path(n != 1)) { + nxt_log_alert(thr->log, "port ready test: %s left %ui announcements " + "on the router port (expected 1)", name, n); + ret = NXT_ERROR; + + } else if (nxt_slow_path(sent->port_msg.stream != expect)) { + nxt_log_alert(thr->log, "port ready test: %s announced the port on " + "stream #%uD, but the start it answers is stream #%uD " + "-- the router matches its registrations on that " + "number, so this answers nobody", name, + sent->port_msg.stream, expect); + ret = NXT_ERROR; + } + + nxt_port_test_run_error_handler(task, router_port); + + if (other_port != NULL) { + nxt_port_test_run_error_handler(task, other_port); + } + + return ret; +} + + +/* + * Issue #271: process->stream is the start RPC the initiator armed, and the + * handler has to retire it exactly when that RPC has been answered -- which + * is when the NEW_PORT announcement carrying the stream has gone out, not + * when the READY state has been set. + * + * The two are not the same moment. The state is set first and the + * announcement is written afterwards, so a READY whose announcement fails + * leaves the initiator with no reply at all; the only thing that can still + * fail its RPC is the stream in the REMOVE_PID this process's death will + * send (src/nxt_router.c:1148-1153). Clearing on the state -- which is + * what nxt_main_process_sigchld_handler() used to do -- throws that away. + * Keeping it forever is the other error the issue names: stream numbers come + * from one counter that every request draws on, so after a wrap an ordinary + * exit retires somebody else's live RPC. + * + * The fixture is a process announcing itself with no queue descriptor, on a + * prototype port: that is a real shape (only libunit attaches a queue, and + * the core processes that announce to main do not), and it keeps the + * announcement free of descriptors, so the peer's queued message can be + * drained without closing a descriptor another port owns. + * + * The peer is a router, because nxt_proc_send_matrix[] pairs ROUTER with + * PROTOTYPE; without a peer the announcement would have nothing to fail on + * and both cases would pass vacuously. Its port has no socket and is not + * write-ready, so nxt_port_socket_write2() queues rather than writes, and + * nxt_port_test_msg_alloc_failures() can fail that queueing on demand. + */ +static nxt_int_t +nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, + nxt_runtime_t *rt, nxt_mp_t *mp) +{ + uint32_t stream; + nxt_int_t ret; + nxt_bool_t router_got_it; + nxt_port_t *proto_port, *router_port, *main_port; + nxt_process_t *proto, *router, *mainp; + nxt_event_engine_t engine, *saved_engine; + nxt_port_recv_msg_t msg; + + ret = NXT_ERROR; + proto_port = NULL; + router_port = NULL; + main_port = NULL; + stream = 0x21212121; + + proto = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + router = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + mainp = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + + if (nxt_slow_path(proto == NULL || router == NULL || mainp == NULL)) { + return NXT_ERROR; + } + + proto->pid = nxt_pid + 7; + proto->isolated_pid = nxt_pid + 8; + proto->state = NXT_PROCESS_STATE_CREATING; + proto->stream = stream; + nxt_queue_init(&proto->ports); + + nxt_runtime_process_add(task, proto); + + proto_port = nxt_port_new(task, 0, proto->pid, NXT_PROCESS_PROTOTYPE); + if (nxt_slow_path(proto_port == NULL)) { + return NXT_ERROR; + } + + proto_port->pair[0] = -1; + proto_port->pair[1] = -1; + proto_port->socket.fd = -1; + + nxt_queue_insert_tail(&proto->ports, &proto_port->link); + + /* + * A second peer, of main's type. nxt_proc_send_matrix[] pairs MAIN with + * everything, so the prototype announcing a child writes to main as well + * as to the router -- and only the router's write is evidence about the + * start. Without this peer the two rules ("every peer succeeded" and + * "the router succeeded") cannot be told apart. + */ + mainp->pid = nxt_pid + 10; + mainp->isolated_pid = nxt_pid + 10; + mainp->state = NXT_PROCESS_STATE_READY; + nxt_queue_init(&mainp->ports); + + nxt_runtime_process_add(task, mainp); + + main_port = nxt_port_new(task, 0, mainp->pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + nxt_queue_insert_tail(&mainp->ports, &main_port->link); + + router->pid = nxt_pid + 9; + router->isolated_pid = nxt_pid + 9; + router->state = NXT_PROCESS_STATE_READY; + nxt_queue_init(&router->ports); + + nxt_runtime_process_add(task, router); + + router_port = nxt_port_new(task, 0, router->pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + router_port->pair[0] = -1; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + nxt_queue_insert_tail(&router->ports, &router_port->link); + + /* + * An engine only for the announcement: nxt_port_send_port() allocates + * its buffer from task->thread->engine->mem_pool, and the queued message + * is completed on the engine's work queue. The rest of this file needs + * none, so it is installed and taken away again here. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_engine = thr->engine; + thr->engine = &engine; + rt->main_engine = &engine; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port_msg.pid = proto->pid; + msg.port_msg.stream = stream; + msg.fd[0] = -1; + msg.fd[1] = -1; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = proto->isolated_pid; +#endif + + /* + * The announcement cannot be queued. The process is ready as far as the + * state goes, and the initiator has still not been told anything, so the + * stream has to survive -- it is the only thing that will fail that RPC + * when this process dies. + */ + nxt_port_test_msg_alloc_failures(2); + + nxt_port_process_ready_handler(task, &msg); + + nxt_port_test_msg_alloc_failures(0); + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, + NXT_PORT_READY_TEST_NO_MSG, + NULL, + "an announcement whose " + "allocation was made to " + "fail") + != NXT_OK)) + { + goto drain; + } + + if (nxt_slow_path(proto->state != NXT_PROCESS_STATE_READY)) { + nxt_log_alert(thr->log, "port ready test: a queueless prototype was " + "not marked ready"); + goto drain; + } + + if (nxt_slow_path(proto->stream != stream)) { + nxt_log_alert(thr->log, "port ready test: the start stream was " + "cleared although the announcement never went out, so " + "nothing is left to fail that RPC"); + goto drain; + } + + /* + * The same message again, with the announcement allowed through. Now + * the initiator has its reply -- nxt_router_new_port_handler() feeds a + * stream-bearing NEW_PORT to nxt_port_rpc_handler() -- and the stream is + * spent: keeping it would put it into this process's REMOVE_PID and, + * once the counter has wrapped, fail an unrelated live RPC. + */ + nxt_port_process_ready_handler(task, &msg); + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, stream, NULL, + "a healthy READY") + != NXT_OK)) + { + goto drain; + } + + if (nxt_slow_path(proto->stream != 0)) { + nxt_log_alert(thr->log, "port ready test: the start stream survived " + "an announcement that answered it, so a later " + "REMOVE_PID would fail whatever RPC now holds that " + "number"); + goto drain; + } + + /* + * A child naming a stream of its own choosing. The gate above + * authenticates who sent a PROCESS_READY, not what it says, and the + * stream field is written by the sender: a child that echoes something + * other than the stream it was started for -- malformed, or forged by a + * compromised worker -- must not be able to redirect the answer. + * + * The announcement has to carry the stream this process registered when + * it forked the child, so the router retires the start that is really + * pending; and clearing is then correct, because that start has been + * answered. Taking the wire value instead would announce a number no + * registration waits on while clearing the one that does -- the start + * would never be retired by the reply, and the REMOVE_PID fallback would + * carry nothing. + */ + proto->stream = stream; + proto->state = NXT_PROCESS_STATE_CREATING; + msg.port_msg.stream = 0xdeadbeef; + + nxt_port_process_ready_handler(task, &msg); + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, stream, NULL, + "a READY naming a foreign " + "stream") + != NXT_OK)) + { + goto drain; + } + + if (nxt_slow_path(proto->stream != 0)) { + nxt_log_alert(thr->log, "port ready test: a READY naming a foreign " + "stream did not retire the start that was answered"); + goto drain; + } + + /* + * The same with a stream of 0, which is what a truncated or zeroed + * message carries. 0 is the "no RPC" value everywhere else + * (nxt_router_remove_pid_handler() returns on it), so a handler that + * passed the wire value straight through would announce nothing + * answerable and still count the start as replied to. + */ + proto->stream = stream; + proto->state = NXT_PROCESS_STATE_CREATING; + msg.port_msg.stream = 0; + + nxt_port_process_ready_handler(task, &msg); + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, stream, NULL, + "a READY naming stream 0") + != NXT_OK)) + { + goto drain; + } + + if (nxt_slow_path(proto->stream != 0)) { + nxt_log_alert(thr->log, "port ready test: a READY naming stream 0 did " + "not retire the start that was answered"); + goto drain; + } + + /* + * One peer's announcement fails and the other's succeeds. Only the + * router's is evidence about the start: it owns the registration for + * every start, whether main or the prototype did the forking, and + * nxt_router_new_port_handler() retires it on a stream-bearing NEW_PORT. + * A failure to any other peer is a delivery problem for that peer. + * + * One armed allocation failure lands on whichever peer the runtime walks + * first, and that is the order they were registered in, not the pids: + * main is added to the fixture before the router above so that the + * failure falls on main and the router's write goes through. That is + * the branch which discriminates the two implementations -- aggregating + * every peer keeps a stream the router has already retired -- and it is + * reached on every run; the log line below records which branch was + * taken, so a future change to the walk order shows up as a note rather + * than as a test that silently stopped testing anything. + * + * The assertion is still written as the rule for both branches, so it + * cannot fail spuriously if that order does change. The opposite branch + * -- the router itself missing out -- is covered deterministically by + * the failed-announcement case above, which arms a failure for every + * peer. + */ + proto->stream = stream; + proto->state = NXT_PROCESS_STATE_CREATING; + msg.port_msg.stream = stream; + + nxt_port_test_msg_alloc_failures(1); + + nxt_port_process_ready_handler(task, &msg); + + nxt_port_test_msg_alloc_failures(0); + + router_got_it = 0; + + if (nxt_slow_path(nxt_port_ready_test_announced(thr, task, router_port, + main_port, + NXT_PORT_READY_TEST_ANY, + &router_got_it, + "a partly failed " + "announcement") + != NXT_OK)) + { + goto drain; + } + + nxt_log_error(NXT_LOG_INFO, thr->log, "port ready test: the partly failed " + "announcement %s the router", + router_got_it ? "reached" : "missed"); + + if (nxt_slow_path(router_got_it && proto->stream != 0)) { + nxt_log_alert(thr->log, "port ready test: the router was announced to " + "and retired the start, but the stream was kept because " + "another peer's write failed -- after a counter wrap " + "that stream fails an unrelated RPC"); + goto drain; + } + + if (nxt_slow_path(!router_got_it && proto->stream != stream)) { + nxt_log_alert(thr->log, "port ready test: the router was not " + "announced to, so nothing retired the start, yet the " + "stream was cleared -- the REMOVE_PID fallback is gone"); + goto drain; + } + + msg.port_msg.stream = stream; + + ret = NXT_OK; + +drain: + + /* + * A queued message is malloc'd and holds a port reference; the error + * handler is what releases both, and it has to run while the engine it + * was queued on is still installed. The checker above drains after each + * case; this catches whatever a failed case left behind. + */ + nxt_port_test_run_error_handler(task, router_port); + + if (main_port != NULL) { + nxt_port_test_run_error_handler(task, main_port); + } + + thr->engine = saved_engine; + rt->main_engine = NULL; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + +done: + + if (main_port != NULL) { + nxt_port_close(task, main_port); + } + + if (router_port != NULL) { + nxt_port_close(task, router_port); + } + + if (proto_port != NULL) { + nxt_port_close(task, proto_port); + } + + return ret; +} + + nxt_int_t nxt_port_ready_test(nxt_thread_t *thr) { @@ -1177,6 +1635,15 @@ nxt_port_ready_test(nxt_thread_t *thr) #endif + /* + * Last, because it registers two more processes in the fixture runtime + * and every case above walks that runtime when it announces a port. + */ + ret = nxt_port_ready_test_stream(thr, task, rt, mp); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + done: /* diff --git a/src/test/nxt_router_new_port_test.c b/src/test/nxt_router_new_port_test.c index f9e13f59d..95dc095b1 100644 --- a/src/test/nxt_router_new_port_test.c +++ b/src/test/nxt_router_new_port_test.c @@ -76,7 +76,7 @@ nxt_router_new_port_test(nxt_thread_t *thr) nxt_buf_t buf; nxt_task_t *task; nxt_int_t ret; - nxt_port_t *port, *reply_port; + nxt_port_t *port, *port2, *reply_port; nxt_runtime_t *rt, *saved_rt; nxt_event_engine_t engine; nxt_port_recv_msg_t msg; @@ -89,6 +89,7 @@ nxt_router_new_port_test(nxt_thread_t *thr) sock = -1; queue = -1; port = NULL; + port2 = NULL; reply_port = NULL; task = thr->task; @@ -243,6 +244,119 @@ nxt_router_new_port_test(nxt_thread_t *thr) goto done; } + /* + * An application's main port (id 0) announced with no start stream. The + * handler has to tell the two ways that can happen apart, so both are + * exercised here. + * + * First the duplicate: register (pid + 2, 0) with a stream, then send + * that exact port again with none, as a worker that called + * nxt_unit_ready() twice does once its prototype has retired the start + * stream. The port must survive unchanged -- falling through would + * re-add it to the application hash and ack it a second time. + */ + + nxt_memzero(&new_port, sizeof(nxt_port_msg_new_port_t)); + + new_port.pid = nxt_pid + 2; + new_port.id = 0; + new_port.type = NXT_PROCESS_APP; + new_port.max_size = 1024; + new_port.max_share = 1024; + + nxt_memzero(&buf, sizeof(nxt_buf_t)); + + buf.mem.pos = (u_char *) &new_port; + buf.mem.free = buf.mem.pos + sizeof(nxt_port_msg_new_port_t); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = &buf; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port = reply_port; + msg.port_msg.stream = 5; + msg.port_msg.pid = new_port.pid; + + nxt_router_new_port_handler(task, &msg); + + port2 = msg.u.new_port; + + if (port2 == NULL || !msg.new_port_created) { + nxt_log_alert(thr->log, "router new port test: the first main port " + "announcement did not create a port"); + goto done; + } + + buf.mem.pos = (u_char *) &new_port; + buf.mem.free = buf.mem.pos + sizeof(nxt_port_msg_new_port_t); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = &buf; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port = reply_port; + msg.port_msg.stream = 0; + msg.port_msg.pid = new_port.pid; + + nxt_router_new_port_handler(task, &msg); + + if (msg.u.new_port != port2 || msg.new_port_created) { + nxt_log_alert(thr->log, "router new port test: the repeated " + "announcement did not resolve to the registered port"); + goto done; + } + + if (nxt_runtime_port_find(rt, new_port.pid, 0) != port2) { + nxt_log_alert(thr->log, "router new port test: the repeated " + "announcement disturbed the registered port"); + goto done; + } + + if (port2->app != NULL) { + nxt_log_alert(thr->log, "router new port test: the repeated " + "announcement enrolled the port in an application"); + goto done; + } + + /* + * Then the other case: a main port the router has never seen, announced + * with no stream. No start is waiting on it and nothing else will + * complete it, so the handler must undo the registration this message + * caused rather than leave an orphan in the runtime. + * + * What this pins, honestly: with the check removed both cases reach + * nxt_assert(port->id != 0) -- an abort in a debug build, compiled out in + * a release one, where the orphan check below is what reports it. So the + * case has teeth either way. What it does not cover is the other half of + * the duplicate: ->port_hash_count being incremented past a declined + * nxt_port_hash_add(), which needs an nxt_app_t the fixture does not + * build. + */ + + new_port.pid = nxt_pid + 3; + + buf.mem.pos = (u_char *) &new_port; + buf.mem.free = buf.mem.pos + sizeof(nxt_port_msg_new_port_t); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = &buf; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port = reply_port; + msg.port_msg.stream = 0; + msg.port_msg.pid = new_port.pid; + + nxt_router_new_port_handler(task, &msg); + + if (nxt_runtime_port_find(rt, new_port.pid, 0) != NULL) { + nxt_log_alert(thr->log, "router new port test: a streamless first " + "announcement was left registered"); + goto done; + } + ret = NXT_OK; nxt_log_error(NXT_LOG_NOTICE, thr->log, "router new port test passed"); @@ -270,6 +384,11 @@ nxt_router_new_port_test(nxt_thread_t *thr) nxt_runtime_port_remove(task, port); } + if (port2 != NULL) { + nxt_port_close(task, port2); + nxt_runtime_port_remove(task, port2); + } + if (reply_port != NULL) { nxt_port_close(task, reply_port); nxt_runtime_port_remove(task, reply_port); From bdbdba86d5c9c271d4126c2b2b47f6afd59d0308 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 01:53:40 +0200 Subject: [PATCH 108/391] runtime: finish a short pid-file write, and close the file on failure nxt_runtime_pid_file_create() (src/nxt_runtime.c:1564) wrote the pid with a single nxt_file_write() and treated anything short of the whole buffer as a failure: if (nxt_file_write(&file, pid, length, 0) != length) { return NXT_ERROR; } nxt_file_close(task, &file); Two things are wrong with that. pwrite() is allowed to store less than it was asked for, and a partial store here is recoverable -- resuming from where it stopped writes a correct pid file where this reports a failure that aborts the runtime. The buffer is at most twenty-odd bytes, so a short write needs a signal or a full filesystem to happen at all, which is why it has never been seen; it is still the wrong answer when it does. The second is not conditional on that. The early return leaves file.fd open: nxt_file_close() is only reached on the success path, so every failing write leaks the descriptor. The process usually exits soon afterwards and the kernel reclaims it, but the leak is real and the function reads as if it closes what it opened. Loop until the buffer is stored, the way nxt_main_file_store() does since 9c0ab65c, and close the file on both paths. A zero return carries no errno and cannot make progress, so it ends the loop with an alert naming how much was stored. The open result moves from "n" to "ret" so that "n" can be the ssize_t the write returns; nxt_int_t and ssize_t are the same width here, but the comparison against a byte count wants the signed size type. Verified by running unitd with an explicit --pid: the file names the live "unit: main" process. Co-Authored-By: Claude Opus 5 --- src/nxt_runtime.c | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 59013efbf..e8dcabfa6 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -1563,8 +1563,9 @@ nxt_current_directory(nxt_mp_t *mp) static nxt_int_t nxt_runtime_pid_file_create(nxt_task_t *task, nxt_file_name_t *pid_file) { - ssize_t length; - nxt_int_t n; + size_t size, written; + ssize_t n; + nxt_int_t ret; nxt_file_t file; u_char pid[NXT_INT64_T_LEN + nxt_length("\n")]; @@ -1574,17 +1575,35 @@ nxt_runtime_pid_file_create(nxt_task_t *task, nxt_file_name_t *pid_file) nxt_fs_mkdir_p_dirname(pid_file, 0755); - n = nxt_file_open(task, &file, O_WRONLY, O_CREAT | O_TRUNC, - NXT_FILE_DEFAULT_ACCESS); + ret = nxt_file_open(task, &file, O_WRONLY, O_CREAT | O_TRUNC, + NXT_FILE_DEFAULT_ACCESS); - if (n != NXT_OK) { + if (ret != NXT_OK) { return NXT_ERROR; } - length = nxt_sprintf(pid, pid + sizeof(pid), "%PI%n", nxt_pid) - pid; + size = nxt_sprintf(pid, pid + sizeof(pid), "%PI%n", nxt_pid) - pid; - if (nxt_file_write(&file, pid, length, 0) != length) { - return NXT_ERROR; + /* + * pwrite() may store less than it was asked for, so resume from where it + * stopped rather than report a short write as a failure. A zero return + * carries no errno and cannot make progress, so it ends the loop. + */ + + for (written = 0; written < size; written += n) { + n = nxt_file_write(&file, pid + written, size - written, written); + + if (nxt_slow_path(n <= 0)) { + /* nxt_file_write() logs the errno; a zero return has none. */ + if (n == 0) { + nxt_alert(task, "write(\"%FN\") stored %uz of %uz bytes", + file.name, written, size); + } + + nxt_file_close(task, &file); + + return NXT_ERROR; + } } nxt_file_close(task, &file); From feeeea376aad28c4c1fa17145a62367394931d72 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 02:01:09 +0200 Subject: [PATCH 109/391] string: do not call mem* with a null pointer and a zero length freeunitorg/freeunit#200. A zero-length nxt_str_t carries a NULL start, and three helpers hand that straight to a libc function whose pointer parameters are declared non-null regardless of the length: - nxt_strstr_eq() compares with memcmp() once the lengths match, so two empty strings reach it with two NULL pointers; - nxt_cpymem() calls memcpy() before it can know the copy is empty; - nxt_is_tstr() scans with memchr() twice, once per character it looks for. Every implementation returns without dereferencing anything, which is why this has never misbehaved. It is still undefined, and the nonnull attribute is the kind a compiler is entitled to propagate: having seen the call, it may infer the pointer cannot be NULL and drop a later check that says otherwise. UBSan reports all three, and .github/sanitizer-allowlist.txt waives them -- one entry per site, each pointing here. Guard the calls instead. Two empty strings are equal, dst + 0 is dst, and an empty string contains no character, so the guards return what the calls would have returned. nxt_cpymem() is a hot helper and this adds a branch to it. The test is on a value the caller usually knows at compile time, so most call sites should fold it away, but this is not claimed to be free -- it is claimed to be correct, and cheap enough for what it removes. The three allowlist entries are deliberately left in place. An entry that matches nothing costs nothing, and removing them wants a sanitizer run that demonstrably exercised these paths; the attempts here kept failing for environmental reasons (no pytest for the host interpreter, an occupied :8080, no network inside the isolated container) rather than producing that evidence. Sweep them once a sanitize leg on this branch comes back without the reports. Co-Authored-By: Claude Opus 5 --- src/nxt_string.h | 18 +++++++++++++++++- src/nxt_tstr.h | 5 +++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/nxt_string.h b/src/nxt_string.h index a5bb1f79c..06201096a 100644 --- a/src/nxt_string.h +++ b/src/nxt_string.h @@ -58,6 +58,15 @@ NXT_EXPORT void nxt_memcpy_upcase(u_char *dst, const u_char *src, nxt_inline void * nxt_cpymem(void *dst, const void *src, size_t length) { + /* + * memcpy() declares "src" non-null even for a zero-length copy, and + * callers reach here with the NULL start of an empty nxt_str_t. The + * copy has nothing to do in that case and dst + 0 is dst. + */ + if (length == 0) { + return dst; + } + return memcpy(dst, src, length) + length; } @@ -122,9 +131,16 @@ NXT_EXPORT nxt_str_t *nxt_str_dup(nxt_mp_t *mp, nxt_str_t *dst, NXT_EXPORT char *nxt_str_cstrz(nxt_mp_t *mp, const nxt_str_t *src); +/* + * memcmp() declares both pointers non-null whatever the length, and a + * zero-length nxt_str_t carries a NULL start, so the length test has to + * short-circuit the call rather than only decide its result. Two empty + * strings are equal, which is what falling through to "true" gives. + */ #define nxt_strstr_eq(s1, s2) \ (((s1)->length == (s2)->length) \ - && (memcmp((s1)->start, (s2)->start, (s1)->length) == 0)) + && ((s1)->length == 0 \ + || memcmp((s1)->start, (s2)->start, (s1)->length) == 0)) #define nxt_strcasestr_eq(s1, s2) \ diff --git a/src/nxt_tstr.h b/src/nxt_tstr.h index 7fb02631e..7c067cdba 100644 --- a/src/nxt_tstr.h +++ b/src/nxt_tstr.h @@ -73,6 +73,11 @@ nxt_is_tstr(nxt_str_t *str) { u_char *p; + /* memchr() declares its pointer non-null; an empty string has none. */ + if (str->length == 0) { + return 0; + } + p = memchr(str->start, '`', str->length); if (p != NULL) { return 1; From 279e1bb5d17d0af0c6a6965688548c7260684021 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 14:59:28 +0200 Subject: [PATCH 110/391] router: escape every value of an object-format access log When "access_log"/"format" is a JSON object, the router only built the log record member-by-member (nxt_router_access_log_json(), building a conf object in the request pool and printing it with nxt_conf_json_print(), whose nxt_conf_json_escape() escapes quotes, backslashes and control characters) if some member value contained a njs expression. Otherwise nxt_router_access_log_format_create() serialised the whole configured object to JSON once, at configuration time (src/nxt_router_access_log.c:232 before this change), and compiled the resulting text as a single template string. Variables such as $header_user_agent were then substituted into that already-serialised JSON verbatim at request time, so an attacker-controlled header value containing `", "injected": "1` closed the member and forged new ones, letting a remote client write arbitrary members into the access log. Always take the per-member path for an object format. The njs path is unchanged; it is now simply the only path. The cost is one small conf object per request, allocated from the request memory pool and freed with it. The string form of "format" is a template by design and is not affected. Co-Authored-By: Claude Opus 5 --- src/nxt_router_access_log.c | 68 +++++++++++-------------------------- test/test_access_log.py | 68 +++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 49 deletions(-) diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index 1ec1a2d2c..c88553b84 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -158,10 +158,8 @@ static nxt_router_access_log_format_t * nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_value_t *value) { - size_t size; uint32_t i, n, next; nxt_str_t name, str, *dst; - nxt_bool_t has_js; nxt_conf_value_t *cv; nxt_router_access_log_member_t *member; nxt_router_access_log_format_t *format; @@ -179,68 +177,40 @@ nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, if (value != NULL) { if (nxt_conf_type(value) == NXT_CONF_OBJECT) { - next = 0; - has_js = 0; - n = nxt_conf_object_members_count(value); - for ( ;; ) { + member = nxt_mp_alloc(rtcf->mem_pool, + n * sizeof(nxt_router_access_log_member_t)); + if (nxt_slow_path(member == NULL)) { + return NULL; + } + + next = 0; + + for (i = 0; i < n; i++) { cv = nxt_conf_next_object_member(value, &name, &next); if (cv == NULL) { break; } - nxt_conf_get_string(cv, &str); - - if (nxt_tstr_is_js(&str)) { - has_js = 1; - } - } - - if (has_js) { - member = nxt_mp_alloc(rtcf->mem_pool, - n * sizeof(nxt_router_access_log_member_t)); - if (nxt_slow_path(member == NULL)) { + dst = nxt_str_dup(rtcf->mem_pool, &member[i].name, &name); + if (nxt_slow_path(dst == NULL)) { return NULL; } - next = 0; - - for (i = 0; i < n; i++) { - cv = nxt_conf_next_object_member(value, &name, &next); - if (cv == NULL) { - break; - } - - dst = nxt_str_dup(rtcf->mem_pool, &member[i].name, &name); - if (nxt_slow_path(dst == NULL)) { - return NULL; - } - - nxt_conf_get_string(cv, &str); + nxt_conf_get_string(cv, &str); - member[i].tstr = nxt_tstr_compile(rtcf->tstr_state, &str, - NXT_TSTR_LOGGING); - if (nxt_slow_path(member[i].tstr == NULL)) { - return NULL; - } + member[i].tstr = nxt_tstr_compile(rtcf->tstr_state, &str, + NXT_TSTR_LOGGING); + if (nxt_slow_path(member[i].tstr == NULL)) { + return NULL; } - - format->nmembers = n; - format->member = member; - - return format; } - size = nxt_conf_json_length(value, NULL); - - str.start = nxt_mp_nget(rtcf->mem_pool, size); - if (nxt_slow_path(str.start == NULL)) { - return NULL; - } + format->nmembers = n; + format->member = member; - str.length = nxt_conf_json_print(str.start, value, NULL) - - str.start; + return format; } else { nxt_conf_get_string(value, &str); diff --git a/test/test_access_log.py b/test/test_access_log.py index b5e1e3f9a..ac2ce569b 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -1,3 +1,4 @@ +import json import time import pytest @@ -307,6 +308,73 @@ def check_format(log_format, expect, url='/'): check_format(log_format, '{"status":"200","uri":"/"}') +def _read_log_lines(): + with open( + f'{option.temp_dir}/access.log', 'r', encoding='utf-8', errors='strict' + ) as f: + return [ln for ln in f.read().splitlines() if ln] + + +def test_access_log_object_escape(wait_for_record): + load('empty') + + # Every character below is significant inside a JSON string: the first + # chunk closes the "ua" member and opens a forged one, then come a bare + # quote, a backslash, a control character (HTAB is legal in a field + # value) and a multi-byte UTF-8 sequence. + evil = '", "injected": "1 " \\ \t \u00e9' + + set_format({'status': '$status', 'ua': '$header_user_agent'}) + + assert ( + client.get( + headers={ + 'Host': 'localhost', + 'User-Agent': evil, + 'Connection': 'close', + } + )['status'] + == 200 + ) + assert wait_for_record(r'"status":"200"', 'access.log') is not None + + line = _read_log_lines()[-1] + + record = json.loads(line) + + assert list(record.keys()) == ['status', 'ua'], 'no injected members' + assert record['status'] == '200' + assert record['ua'] == evil, 'value round-trips' + + +def test_access_log_object_escape_njs(require, wait_for_record): + require({'modules': {'njs': 'any'}}) + + load('empty') + + evil = '", "injected": "1 " \\ \t \u00e9' + + set_format({'uri': '`${vars.uri}`', 'ua': '$header_user_agent'}) + + assert ( + client.get( + headers={ + 'Host': 'localhost', + 'User-Agent': evil, + 'Connection': 'close', + } + )['status'] + == 200 + ) + assert wait_for_record(r'"uri":"/"', 'access.log') is not None + + record = json.loads(_read_log_lines()[-1]) + + assert list(record.keys()) == ['uri', 'ua'], 'no injected members' + assert record['uri'] == '/' + assert record['ua'] == evil, 'value round-trips' + + def test_access_log_variables(wait_for_record): load('mirror') From 420f4b2d8e32915c50c7c1bd07a301db1a749baa Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 5 Sep 2026 17:22:27 +0200 Subject: [PATCH 111/391] socket: refuse a port message whose control data was truncated recvmsg() sets MSG_CTRUNC when it could not deliver all of the control data the sender attached. Nothing in src/ looked at msg_flags: nxt_recvmsg() (src/nxt_socket_msg.c:37) copied msg_controllen into oob->size and returned, and nxt_socket_msg_oob_get() parsed whatever survived. The case that matters is an SCM_RIGHTS the kernel discards while delivering everything else. scm_recv() puts the SO_PASSCRED credential first and only then calls scm_detach_fds(), which needs a free descriptor slot per fd; under RLIMIT_NOFILE pressure it installs none and raises MSG_CTRUNC. What then reaches a handler is a message that passes every check it makes -- right size, right type, and on Linux a kernel-validated sender pid -- whose fd is simply -1. Each fd-carrying type takes some fd-less path from there: nxt_port_new_port_handler() (src/nxt_port.c) builds a port with pair[1] == -1, nxt_port_mmap_handler() has no segment to map, nxt_port_change_log_file_handler() redirects nothing, and nxt_port_process_ready_handler() marks a process ready with no queue. Record the flag in nxt_recv_oob_t and make both accessors report it as a receive error. The check follows the parse loop rather than preceding it: a truncated control block can still have delivered a descriptor -- put_cmsg() installs what fits before setting MSG_CTRUNC -- and the caller can only close what it has been told about. Both nxt_port_socket.c call sites already close msg.fd and fail the message on a non-OK return, so the descriptors that did arrive are reclaimed and the message is dropped exactly the way a malformed one is; nxt_unit_process_msg() (src/nxt_unit.c:1087) likewise closes through its "done" label. The two receive sites fail a refused message differently, and that asymmetry is master's, not this change's. nxt_port_read_handler() (src/nxt_port_socket.c:868) does "goto fail", which routes to nxt_port_error_handler(): the whole port goes down, and the refcount drop takes it through nxt_port_rpc_close() (src/nxt_port.c:163), failing every RPC registered on it. nxt_port_queue_read_handler() (src/nxt_port_socket.c:1068) drops the one message and leaves the port live. Keeping those dispositions is deliberate: a truncated message is not more trustworthy than a malformed one, and giving it a third disposition would change how every existing oob failure is handled under cover of a truncation fix. Nothing wedges either way -- on the socket path rpc_close() fails the pending RPCs, and on the queue path the message that can carry a start reply is a socket message, since a queue message carries no descriptors at all (nxt_port_queue_read_handler() zeroes msg.fd up front). Dropping the message at the queue site does, however, have to give back what the iteration took, and master's bare return did not. queue->nitems is incremented on entry to the handler and is read by nxt_port_queue_send() in every other process holding the port: while it is non-zero the sender skips the socket notification, because a reader is supposed to be awake. Returning without decrementing it left the counter raised for good, so the port was never woken for a queue-only message again -- one refused datagram costing the port every message after it. The READ_SOCKET marker counted in port->from_socket and the buffer from nxt_port_buf_alloc() leaked on the same return. All three are now unwound before the return, in the shape the buf-alloc failure branch just above already uses. This corrects master's behaviour on its own oob-failure path as well; it is folded in here because this change is what makes that path reachable from a condition a peer does not control. nxt_port_rpc_error() from #280 would allow failing the single stream instead of the port; that PR is not on master and nothing here depends on it. The truncation is not self-inflicted. NXT_OOB_RECV_SIZE (src/nxt_socket_msg.h:29) is CMSG_SPACE(2 * sizeof(int)) plus the credential cmsg, and two is the whole of what Unit can send: nxt_port_recv_msg_t has two fd slots and nxt_socket_msg_oob_init() attaches at most that many. The buffer is already the maximum, so it is left alone and the new comment there records why sizing cannot be the remedy. The new member goes after "buf", not before it, and that placement is load bearing. CMSG_FIRSTHDR()/CMSG_NXTHDR() walk "buf" as struct cmsghdr, and nothing in the declaration states that alignment: it comes from "buf" following a size_t directly, alignof(struct cmsghdr) being alignof(size_t) on the ABIs Unit builds for. A first version of this put "truncated" between the two, which -- nxt_bool_t being nxt_uint_t -- moved "buf" from offset 8 to offset 12 and made every cmsg access undefined; UBSan reported six of them, "member access within misaligned address ... for type 'struct cmsghdr', which requires 8 byte alignment", from bits/socket.h in the sanitize legs. A comment on the struct now says why nothing may be inserted ahead of "buf", and the buffer carries an explicit nxt_aligned(NXT_OOB_ALIGN) as a second line of defence -- second, because nxt_aligned() expands to nothing outside GCC and clang, so field order has to be the guarantee. Both accessors also stop trusting cmsg_len against the bytes that actually arrived. CMSG_FIRSTHDR() checks only that msg_controllen covers a struct cmsghdr, so the first header's cmsg_len can claim more data than was received and the CMSG_DATA() copy would run off the end of oob->buf -- and a block the kernel cut short is precisely where an over-claiming length shows up. The loop now refuses a header whose full extent does not fit in oob->size, which is the fail-closed shape truncation wants anyway. nxt_socket_msg_oob_reset() retires ->size and ->truncated together, and replaces the four "oob.size = 0" assignments in src/nxt_unit.c, so a recycled read buffer cannot present a previous message's MSG_CTRUNC as this one's; nxt_unit_rbuf_cpy() carries the flag with the rest of the control data. The port_recv callback path reports a length rather than msg_flags and so clears the flag explicitly. The two alerts in nxt_port_socket.c are folded into nxt_port_oob_error_alert(), which names truncation separately and by message type and stream: that is what ties the loss to the start, mmap or port hand-off that will not complete, and it is the one thing a reader of the log cannot otherwise infer, since the message looks whole in every other respect. The Go module reaches the same control data through the port_recv callback (go/nxt_cgo_lib.c:60), which reports a length rather than msg_flags, so oob.truncated cannot be plumbed across it without changing the public nxt_unit.h callback ABI. nxt_go_port_recv() (go/port.go:243) discarded the flags word ReadMsgUnix() returns, so a Go application kept the pre-fix behaviour -- NEW_PORT or MMAP with fd -1 -- while every other language module was covered. It now tests MSG_CTRUNC and returns -1, which nxt_unit_port_recv() turns into NXT_UNIT_ERROR at its rbuf->size < 0 arm, before it parses any control data; that is why the callback path still clears oob.truncated rather than honouring it, and the comment there says so. Reporting the error is not enough on its own. Truncation does not mean nothing was delivered -- scm_detach_fds() installs the descriptors that fit and only then raises MSG_CTRUNC -- and those are open in the Go process already, while an error return leaves *oob_size at 0 so libunit never parses the block and never closes them. That is a descriptor leak under exactly the pressure that caused the truncation. closeUnixRights() (go/port.go) parses the delivered bytes with syscall.ParseSocketControlMessage()/ParseUnixRights() and closes what it finds before the error is returned; parse failures are ignored, since the block is known to be damaged and whatever can still be read out of it is worth closing. src/test/nxt_port_ctrunc_test.c drives the truncation through the receive buffer's length rather than through RLIMIT_NOFILE -- the other input to the same kernel decision, deterministic and confined to the test. Over a socketpair from nxt_socketpair_create(), so that SO_PASSCRED reproduces the production ordering, a two-descriptor message is received three ways: with room for the credential alone, with room for the credential and one of the two descriptors, and with the full NXT_OOB_RECV_SIZE. The first two assert that both accessors return NXT_ERROR, that they agree on the descriptors delivered, and that the open-descriptor count is unchanged once those are closed; the third asserts the intact message is still accepted with both descriptors and the sender's pid. Without this change the first two cases fail: the accessors return NXT_OK with fd[0] == -1 and a valid credential, which is the bug. The two short-buffer cases assert a kernel behaviour that Linux and the BSDs share and xnu does not: scm_detach_fds() installs what fits and raises MSG_CTRUNC, whereas Darwin externalizes the rights before it measures the buffer, so the same short receive there reports no truncation and an unconditional assertion would simply fail. They are compiled out under NXT_MACOSX with a notice naming the reason. What they prove -- that a real kernel produces the flag -- is all that is lost; the accessor contract itself is covered on every platform by a third case that takes a control block delivered whole, marks it truncated by hand, and requires the same refusal with both descriptors still reported. That is also the production shape, since a real truncation can deliver every descriptor asked for and still raise the flag. A fourth case covers the queue-backed read path end to end: a port with a mapped nxt_port_queue_t, its handler installed through nxt_port_read_enable(), and a message refused after its descriptor was delivered. The refusal is driven by a missing SCM_CREDENTIALS rather than by truncation -- the handler sizes its own receive buffer, so a real MSG_CTRUNC there would mean exhausting the test process's descriptor table, which is neither deterministic nor confined to the test -- but it reaches the identical cleanup path. The case asserts that queue->nitems and port->from_socket come back to their entry values and, the point of the counter, that the very next message on that port is still delivered. Without the cleanup the counter stays raised; the case is compiled only where credential passing exists, which is what makes the refusal deterministic. Both negative controls were run: with the accessor guards removed the marked case fails ("oob_get accepted a block marked truncated"), and with the queue cleanup removed the queue case fails ("a refused message left queue->nitems at 1, was 0"). The suite was also run under -fsanitize=address,undefined. Branch and origin/master come out identical: one runtime error each, the pre-existing src/nxt_buf.h:250 "null pointer passed as argument 2" reached from nxt_router_proto_wedge_test, and no report anywhere in bits/socket.h or nxt_socket_msg.c. Fixes #264. Restoring the counter is necessary but not sufficient, so the refusal keeps draining rather than returning. queue->nitems is non-zero for the whole invocation, and that is exactly how nxt_port_queue_send() decides a reader is awake and skips the socket notification; a message enqueued in that window is therefore never announced, and returning here left nobody to come back for it. nitems is deliberately not decremented on this path any more: the handler holds one credit for the invocation and whichever exit it takes releases it, so unwinding here as well gave one credit back twice and underflowed the counter to UINT32_MAX -- a worse wedge than the one being fixed. The new queue case pins all three shapes; each fails differently. nxt_port_ctrunc_test_truncated() is now defined only in the arm that calls it: both call sites are in the non-macOS branch, and an unused static trips -Wunused-function under -Werror on a platform no CI leg builds. The queue case is gated on NXT_HAVE_UCRED rather than NXT_CRED_USECMSG. It builds the message the receiver has to refuse out of a credential that is enforced but not attached -- true on Linux with SO_PASSCRED off the socketpair, false where the credential travels as SCM_CREDS, because there nxt_socket_msg_oob_init() attaches one itself and the same call yields a valid message. The truncation cases keep their wider gate. MSG_CTRUNC is POSIX and the flag is available wherever Unit builds, but what a kernel does with the descriptors it could not fit is not uniform. Only the Linux behaviour -- install what fits, then raise the flag, which is what makes reporting the arrived descriptors necessary -- is verified here, so the header says so rather than describing it as what MSG_CTRUNC means everywhere. The refusal does not depend on which way a platform goes; on one that discards the rights instead, fd[] simply stays -1 and the same code is correct. Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + go/port.go | 52 +- src/nxt_port_socket.c | 79 ++- src/nxt_socket_msg.c | 6 + src/nxt_socket_msg.h | 114 +++- src/nxt_unit.c | 35 +- src/test/nxt_port_ctrunc_test.c | 960 ++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 9 files changed, 1237 insertions(+), 15 deletions(-) create mode 100644 src/test/nxt_port_ctrunc_test.c diff --git a/auto/sources b/auto/sources index c2d57edf8..32297b797 100644 --- a/auto/sources +++ b/auto/sources @@ -188,6 +188,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ src/test/nxt_port_change_file_test.c \ + src/test/nxt_port_ctrunc_test.c \ src/test/nxt_port_fd_test.c \ " diff --git a/go/port.go b/go/port.go index ed8283fe3..020289ce0 100644 --- a/go/port.go +++ b/go/port.go @@ -215,6 +215,29 @@ func nxt_go_port_send(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, return C.ssize_t(n) } +// closeUnixRights closes every descriptor an SCM_RIGHTS block carries. It is +// used on paths that refuse a message after the kernel has already installed +// its descriptors, where nothing else will ever see them. Parse failures are +// ignored deliberately: the block is known to be damaged, and whatever can +// still be read out of it is worth closing. +func closeUnixRights(oob []byte) { + msgs, err := syscall.ParseSocketControlMessage(oob) + if err != nil { + return + } + + for _, m := range msgs { + fds, err := syscall.ParseUnixRights(&m) + if err != nil { + continue + } + + for _, fd := range fds { + syscall.Close(fd) + } + } +} + //export nxt_go_port_recv func nxt_go_port_recv(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, oob unsafe.Pointer, oob_size *C.size_t) C.ssize_t { @@ -240,9 +263,36 @@ func nxt_go_port_recv(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, return 0 } - n, oobn, _, _, err := p.rcv.ReadMsgUnix(GoBytes(buf, buf_size), + n, oobn, flags, _, err := p.rcv.ReadMsgUnix(GoBytes(buf, buf_size), GoBytes(oob, oob_capacity)) + // The control data was cut short, so a descriptor this message was + // meant to carry may not be here; libunit would take its fd-less path + // on a message that otherwise looks whole. The callback reports a + // length rather than msg_flags, so there is no way to hand the + // truncation across as such: report it as a read error, which + // nxt_unit_port_recv() turns into NXT_UNIT_ERROR before it parses any + // control data. This is the one place the Go wrapper can see + // MSG_CTRUNC at all -- ReadMsgUnix is what consumed it. + if err == nil && flags&syscall.MSG_CTRUNC != 0 { + // Truncation does not imply that nothing was delivered: + // scm_detach_fds() installs the descriptors that fit and only + // then raises MSG_CTRUNC. Those are open in this process + // already, and reporting an error leaves *oob_size at 0, so + // libunit never parses the block and never closes them -- + // a descriptor leak under exactly the pressure that caused + // the truncation. Close them here instead; best effort, + // since a block the kernel cut may not parse. + if oobn > 0 { + closeUnixRights(GoBytes(oob, C.int(oobn))) + } + + nxt_go_alert("control data truncated on a %d byte message; "+ + "message dropped", n) + + return C.ssize_t(-1) + } + if err != nil { if nerr, ok := err.(*net.OpError); ok { if nerr.Err == io.EOF { diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 40bbc6860..84dd1b37c 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -765,6 +765,45 @@ nxt_port_read_close(nxt_port_t *port) } +/* + * Report a message whose control data could not be taken, on the way to + * dropping it. + * + * Truncation is called out separately because it is the case a reader of the + * log cannot otherwise infer: the payload and, on Linux, the kernel + * credential arrived intact, and only the SCM_RIGHTS was cut -- so what a + * handler would see is a well-formed, authenticated message whose descriptor + * is -1. The message is named by type and stream, which is what ties the + * loss to the start, mmap or port hand-off that will not complete. + * + * "n" is the payload length recvmsg() returned; below sizeof(nxt_port_msg_t) + * there is no header to name the message by. + */ + +static void +nxt_port_oob_error_alert(nxt_task_t *task, nxt_port_t *port, + nxt_recv_oob_t *oob, nxt_port_recv_msg_t *msg, ssize_t n) +{ + if (nxt_fast_path(!oob->truncated)) { + nxt_alert(task, "failed to get oob data from %d", port->socket.fd); + + return; + } + + if (n >= (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_alert(task, "port{%d,%d} %d: control data truncated on message " + "type %d stream #%uD; message dropped", + (int) port->pid, (int) port->id, port->socket.fd, + (int) msg->port_msg.type, msg->port_msg.stream); + + } else { + nxt_alert(task, "port{%d,%d} %d: control data truncated on a %z byte " + "message; message dropped", + (int) port->pid, (int) port->id, port->socket.fd, n); + } +} + + static void nxt_port_read_handler(nxt_task_t *task, void *obj, void *data) { @@ -826,9 +865,13 @@ nxt_port_read_handler(nxt_task_t *task, void *obj, void *data) ret = nxt_socket_msg_oob_get(&oob, msg.fd, nxt_recv_msg_cmsg_pid_ref(&msg)); if (nxt_slow_path(ret != NXT_OK)) { - nxt_alert(task, "failed to get oob data from %d", - port->socket.fd); + nxt_port_oob_error_alert(task, port, &oob, &msg, n); + /* + * Whatever part of the SCM_RIGHTS did arrive is in msg.fd; + * the dispatcher does not reclaim descriptors once a message + * is refused here, so close them before dropping it. + */ nxt_port_close_fds(msg.fd); goto fail; @@ -1022,12 +1065,38 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) ret = nxt_socket_msg_oob_get(&oob, msg.fd, nxt_recv_msg_cmsg_pid_ref(&msg)); if (nxt_slow_path(ret != NXT_OK)) { - nxt_alert(task, "failed to get oob data from %d", - port->socket.fd); + nxt_port_oob_error_alert(task, port, &oob, &msg, n); + /* See nxt_port_read_handler(). */ nxt_port_close_fds(msg.fd); - return; + /* + * Unwind what this iteration took, then keep draining. + * Returning bare -- which is what this site did -- strands + * the buffer and the READ_SOCKET marker this read was + * consuming, and it strands whatever was queued behind + * them: queue->nitems is non-zero for the whole + * invocation, which is precisely how nxt_port_queue_send() + * decides a reader is already awake and skips the socket + * notification. A message enqueued in that window is + * therefore never announced, and returning here means + * nobody ever comes back for it -- one refused datagram + * costing the port every message queued behind it. + * + * nitems is deliberately not decremented here. The + * handler holds exactly one nitems credit for the whole + * invocation, and whichever exit it finally takes releases + * it -- the empty-queue check above, or the loop's own + * termination. Decrementing here as well as there would + * give back one credit twice. + */ + if (port->from_socket > 0) { + port->from_socket--; + } + + nxt_port_buf_free(port, b); + + continue; } } diff --git a/src/nxt_socket_msg.c b/src/nxt_socket_msg.c index 3b35ab292..5b019970d 100644 --- a/src/nxt_socket_msg.c +++ b/src/nxt_socket_msg.c @@ -47,10 +47,16 @@ nxt_recvmsg(nxt_socket_t s, nxt_iobuf_t *iob, nxt_uint_t niob, msg.msg_control = oob->buf; msg.msg_controllen = sizeof(oob->buf); + /* Cleared so that MSG_CTRUNC below is read from what recvmsg() wrote. */ + msg.msg_flags = 0; + n = recvmsg(s, &msg, 0); if (nxt_fast_path(n != -1)) { oob->size = msg.msg_controllen; + + /* Recorded for nxt_socket_msg_oob_get(); see NXT_OOB_RECV_SIZE. */ + oob->truncated = (msg.msg_flags & MSG_CTRUNC) != 0; } return n; diff --git a/src/nxt_socket_msg.h b/src/nxt_socket_msg.h index 8a36245cc..fcc82139f 100644 --- a/src/nxt_socket_msg.h +++ b/src/nxt_socket_msg.h @@ -42,9 +42,56 @@ typedef struct cmsgcred nxt_socket_cred_t; #endif +/* + * NXT_OOB_RECV_SIZE is the largest control block Unit can legitimately be + * handed: the two descriptors a message may carry -- nxt_port_recv_msg_t + * has exactly two fd slots and nxt_socket_msg_oob_init() never attaches + * more -- plus, where the platform passes credentials, the one credential + * cmsg the kernel or the sender adds. A receive buffer of that size can + * therefore never truncate a well-formed message: MSG_CTRUNC means either + * that the kernel could not deliver all of the control data it had for this + * message (an SCM_RIGHTS it could not install in full, typically under + * RLIMIT_NOFILE pressure), or that a peer sent more control data than any + * Unit message type uses. Both are receive errors, not conditions a + * handler can be expected to notice: what reaches it is a well-formed, + * authenticated message whose descriptor is simply -1. + * + * What a kernel does with the descriptors it could not fit is not uniform, + * and only the Linux behaviour is verified here: scm_detach_fds() installs + * the ones that fit into this process and only then raises MSG_CTRUNC, so + * they are open, owned, and unreachable unless the accessor reports them -- + * which is why nxt_socket_msg_oob_get() and nxt_socket_msg_oob_get_fds() + * fill fd[] before failing the message. A kernel that instead discards the + * rights leaves nothing to leak, and the same code is correct there because + * fd[] simply stays -1. The refusal itself does not depend on which way a + * platform goes; the reporting is what the Linux semantics make necessary. + * MSG_CTRUNC is POSIX, so the flag is available wherever Unit builds, but + * only the Linux side of this is covered by a test (see + * src/test/nxt_port_ctrunc_test.c, whose kernel-driven cases skip on macOS + * and whose queue case is gated on NXT_HAVE_UCRED). + */ + +#define NXT_OOB_ALIGN (sizeof(size_t)) + + +/* + * "buf" holds a control block that CMSG_FIRSTHDR()/CMSG_NXTHDR() walk as + * struct cmsghdr, so its address has to carry that struct's alignment. + * Nothing here states that alignment: it comes from "buf" following a size_t + * directly, since alignof(struct cmsghdr) == alignof(size_t) on the ABIs Unit + * builds for (cmsg_len is a size_t on glibc, socklen_t elsewhere). A field + * inserted between the two moves "buf" off that alignment and every cmsg + * access becomes undefined -- UBSan reports it as "member access within + * misaligned address ... for type 'struct cmsghdr'". Keep new members after + * "buf", as "truncated" is. nxt_aligned() is not a substitute: it expands to + * nothing outside GCC and clang. + */ + typedef struct { - size_t size; - u_char buf[NXT_OOB_RECV_SIZE]; + size_t size; + u_char buf[NXT_OOB_RECV_SIZE] nxt_aligned(NXT_OOB_ALIGN); + /* recvmsg() reported MSG_CTRUNC: the control data is incomplete. */ + nxt_bool_t truncated; } nxt_recv_oob_t; @@ -83,6 +130,21 @@ NXT_CMSG_NXTHDR(struct msghdr *msgh, struct cmsghdr *cmsg) } +/* + * Put a receive buffer into the "carries nothing" state. Callers that fill + * oob themselves, or that recycle a buffer, use this rather than assigning + * ->size alone, so that every field describing the control data is retired + * together. + */ + +nxt_inline void +nxt_socket_msg_oob_reset(nxt_recv_oob_t *oob) +{ + oob->size = 0; + oob->truncated = 0; +} + + nxt_inline void nxt_socket_msg_oob_init(nxt_send_oob_t *oob, int *fds) { @@ -153,6 +215,21 @@ nxt_socket_msg_oob_get_fds(nxt_recv_oob_t *oob, nxt_fd_t *fd) cmsg != NULL; cmsg = NXT_CMSG_NXTHDR(&msg, cmsg)) { + /* + * Fail closed on a header that does not fit the bytes actually + * received. CMSG_FIRSTHDR() only checks that msg_controllen covers + * a struct cmsghdr, so the first header's cmsg_len can still claim + * more data than arrived, and a block the kernel cut short is + * exactly where that happens. Reading CMSG_DATA() over that claim + * would run off the end of oob->buf. + */ + if (nxt_slow_path(cmsg->cmsg_len < CMSG_LEN(0) + || (size_t) ((u_char *) cmsg - oob->buf) + + cmsg->cmsg_len > oob->size)) + { + return NXT_ERROR; + } + size = cmsg->cmsg_len - CMSG_LEN(0); if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { @@ -162,10 +239,19 @@ nxt_socket_msg_oob_get_fds(nxt_recv_oob_t *oob, nxt_fd_t *fd) nxt_memcpy(fd, CMSG_DATA(cmsg), size); - return NXT_OK; + break; } } + /* + * Checked after the loop, not before it: a truncated control block can + * still have delivered a descriptor, and the caller can only close what + * it has been told about. + */ + if (nxt_slow_path(oob->truncated)) { + return NXT_ERROR; + } + return NXT_OK; } @@ -177,7 +263,7 @@ nxt_socket_msg_oob_get(nxt_recv_oob_t *oob, nxt_fd_t *fd, nxt_pid_t *pid) struct msghdr msg; struct cmsghdr *cmsg; - if (oob->size == 0) { + if (oob->size == 0 && !oob->truncated) { return NXT_OK; } @@ -192,6 +278,21 @@ nxt_socket_msg_oob_get(nxt_recv_oob_t *oob, nxt_fd_t *fd, nxt_pid_t *pid) cmsg != NULL; cmsg = NXT_CMSG_NXTHDR(&msg, cmsg)) { + /* + * Fail closed on a header that does not fit the bytes actually + * received. CMSG_FIRSTHDR() only checks that msg_controllen covers + * a struct cmsghdr, so the first header's cmsg_len can still claim + * more data than arrived, and a block the kernel cut short is + * exactly where that happens. Reading CMSG_DATA() over that claim + * would run off the end of oob->buf. + */ + if (nxt_slow_path(cmsg->cmsg_len < CMSG_LEN(0) + || (size_t) ((u_char *) cmsg - oob->buf) + + cmsg->cmsg_len > oob->size)) + { + return NXT_ERROR; + } + size = cmsg->cmsg_len - CMSG_LEN(0); if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { @@ -222,6 +323,11 @@ nxt_socket_msg_oob_get(nxt_recv_oob_t *oob, nxt_fd_t *fd, nxt_pid_t *pid) #endif } + /* See nxt_socket_msg_oob_get_fds() on why this follows the loop. */ + if (nxt_slow_path(oob->truncated)) { + return NXT_ERROR; + } + #if (NXT_CRED_USECMSG) /* For platforms supporting credential passing, it's enforced */ if (nxt_slow_path(*pid == -1)) { diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 296b10bb3..ded5187f7 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -1090,7 +1090,20 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf, rc = nxt_socket_msg_oob_get_fds(&rbuf->oob, recv_msg.fd); if (nxt_slow_path(rc != NXT_OK)) { - nxt_unit_alert(ctx, "failed to receive file descriptor over cmsg"); + if (rbuf->oob.truncated) { + /* + * The descriptor this message was meant to carry may not be + * here: NEW_PORT and MMAP would take their fd-less path on a + * message that otherwise looks whole. Refuse it; "done" closes + * whatever part of the SCM_RIGHTS did arrive. + */ + nxt_unit_alert(ctx, "control data truncated on a %d byte " + "message; message dropped", (int) rbuf->size); + + } else { + nxt_unit_alert(ctx, "failed to receive file descriptor over cmsg"); + } + rc = NXT_UNIT_ERROR; goto done; } @@ -3023,7 +3036,7 @@ nxt_unit_read_buf_get(nxt_unit_ctx_t *ctx) pthread_mutex_unlock(&ctx_impl->mutex); - rbuf->oob.size = 0; + nxt_socket_msg_oob_reset(&rbuf->oob); return rbuf; } @@ -6403,7 +6416,7 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, * a leftover oob.size would offer them again. */ port_impl->socket_rbuf->size = 0; - port_impl->socket_rbuf->oob.size = 0; + nxt_socket_msg_oob_reset(&port_impl->socket_rbuf->oob); nxt_unit_debug(ctx, "port{%d,%d} use suspended message %d", (int) port->id.pid, (int) port->id.id, @@ -6488,7 +6501,7 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_rbuf_cpy(port_impl->socket_rbuf, rbuf); - rbuf->oob.size = 0; + nxt_socket_msg_oob_reset(&rbuf->oob); goto retry; } @@ -6500,6 +6513,7 @@ nxt_unit_rbuf_cpy(nxt_unit_read_buf_t *dst, nxt_unit_read_buf_t *src) memcpy(dst->buf, src->buf, src->size); dst->size = src->size; dst->oob.size = src->oob.size; + dst->oob.truncated = src->oob.truncated; memcpy(dst->oob.buf, src->oob.buf, src->oob.size); } @@ -6592,6 +6606,17 @@ nxt_unit_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, } rbuf->oob.size = oob_size; + /* + * The callback reports a length, not msg_flags, so a truncation + * cannot be carried across as such: a wrapper that sees MSG_CTRUNC + * reports a read error instead, and the rbuf->size < 0 arm above + * returns before any control data is parsed (go/port.go does exactly + * this). Nothing can therefore be inferred about truncation here; + * clear the flag so that a previous message's MSG_CTRUNC, left in + * this recycled buffer, cannot be read as this one's. + */ + rbuf->oob.truncated = 0; + return NXT_UNIT_OK; } @@ -6612,7 +6637,7 @@ nxt_unit_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, * clear it here to keep "size 0 implies no control data" holding on * every exit of this function rather than by inspection of callers. */ - rbuf->oob.size = 0; + nxt_socket_msg_oob_reset(&rbuf->oob); if (err == EINTR) { goto retry; diff --git a/src/test/nxt_port_ctrunc_test.c b/src/test/nxt_port_ctrunc_test.c new file mode 100644 index 000000000..f34a8c0b0 --- /dev/null +++ b/src/test/nxt_port_ctrunc_test.c @@ -0,0 +1,960 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for control-data truncation on a port socket + * (src/nxt_socket_msg.h, src/nxt_socket_msg.c). + * + * recvmsg() sets MSG_CTRUNC when it could not deliver all of the control + * data the sender attached. The case that matters here is an SCM_RIGHTS the + * kernel discards while delivering everything else -- the payload and, on + * Linux, the SCM_CREDENTIALS the receiver's SO_PASSCRED asks for. In + * production that happens under descriptor pressure: scm_detach_fds() cannot + * install the descriptors and drops the whole cmsg. + * + * Nothing used to look at msg_flags, so what reached a handler was a message + * that passed every check it makes -- right size, right type, kernel-validated + * sender -- and simply had fd[0] == -1. NEW_PORT, MMAP and CHANGE_FILE each + * take some fd-less path from there. + * + * The truncation is driven directly rather than through RLIMIT_NOFILE: the + * receive buffer's length is the other input to the same kernel decision, and + * unlike a descriptor limit it is deterministic and leaves the rest of the + * test process alone. What is asserted is the contract every receive path + * shares, that nxt_socket_msg_oob_get() and nxt_socket_msg_oob_get_fds() + * refuse a truncated control block -- and that they refuse it after reporting + * whatever descriptors did arrive, since the caller can only close what it + * has been told about. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +#define NXT_CTRUNC_TEST_FD_MAX 1024 + +/* + * The control space the receiver has to keep for the credential cmsg before + * the SCM_RIGHTS can begin. Zero where the platform passes no credential. + */ +#if (NXT_CRED_USECMSG) +#define NXT_CTRUNC_TEST_CRED_SPACE CMSG_SPACE(sizeof(nxt_socket_cred_t)) +#else +#define NXT_CTRUNC_TEST_CRED_SPACE 0 +#endif + + +static nxt_uint_t +nxt_port_ctrunc_test_fd_count(void) +{ + int fd; + nxt_uint_t n; + + n = 0; + + for (fd = 0; fd < NXT_CTRUNC_TEST_FD_MAX; fd++) { + if (fcntl(fd, F_GETFD) != -1) { + n++; + } + } + + return n; +} + + +/* + * Send one message carrying two descriptors, and receive it with a control + * buffer of exactly "controllen" bytes, filling "oob" the way nxt_recvmsg() + * would. A short buffer is what makes the kernel drop the SCM_RIGHTS and + * raise MSG_CTRUNC. + */ +static nxt_int_t +nxt_port_ctrunc_test_exchange(nxt_thread_t *thr, nxt_socket_t *pair, + size_t controllen, nxt_recv_oob_t *oob, ssize_t *received) +{ + int fds[2]; + u_char payload; + ssize_t n; + nxt_send_oob_t send_oob; + struct iovec iov[1]; + struct msghdr msg; + + fds[0] = open("/dev/null", O_RDONLY); + fds[1] = open("/dev/null", O_RDONLY); + + if (fds[0] == -1 || fds[1] == -1) { + nxt_log_alert(thr->log, "port ctrunc test failed to open /dev/null"); + goto fail; + } + + nxt_socket_msg_oob_init(&send_oob, fds); + + payload = 0x5A; + + iov[0].iov_base = &payload; + iov[0].iov_len = sizeof(payload); + + n = nxt_sendmsg(pair[1], iov, 1, &send_oob); + + if (n != (ssize_t) sizeof(payload)) { + nxt_log_alert(thr->log, "port ctrunc test sendmsg failed %E", + nxt_errno); + goto fail; + } + + /* + * sendmsg() duplicated the descriptors into the socket; this end has no + * further use for them, and leaving them open would hide a leak on the + * receiving side behind a constant offset. + */ + (void) close(fds[0]); + (void) close(fds[1]); + + fds[0] = -1; + fds[1] = -1; + + payload = 0; + + iov[0].iov_base = &payload; + iov[0].iov_len = sizeof(payload); + + nxt_memzero(oob, sizeof(nxt_recv_oob_t)); + + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_iov = iov; + msg.msg_iovlen = 1; + msg.msg_control = oob->buf; + msg.msg_controllen = controllen; + msg.msg_flags = 0; + + n = recvmsg(pair[0], &msg, 0); + + if (n != (ssize_t) sizeof(payload) || payload != 0x5A) { + nxt_log_alert(thr->log, "port ctrunc test recvmsg failed %z %E", + n, nxt_errno); + return NXT_ERROR; + } + + oob->size = msg.msg_controllen; + oob->truncated = (msg.msg_flags & MSG_CTRUNC) != 0; + + *received = n; + + return NXT_OK; + +fail: + + if (fds[0] != -1) { + (void) close(fds[0]); + } + + if (fds[1] != -1) { + (void) close(fds[1]); + } + + return NXT_ERROR; +} + + +static void +nxt_port_ctrunc_test_close_fds(nxt_fd_t *fd) +{ + nxt_uint_t i; + + for (i = 0; i < 2; i++) { + if (fd[i] != -1) { + (void) close(fd[i]); + fd[i] = -1; + } + } +} + + +/* + * Defined only where it is called: both call sites sit in the non-macOS arm + * below, and an unused static trips -Wunused-function under -Werror on a + * platform no CI leg builds. + */ + +#if !(NXT_MACOSX) + +/* + * The regression itself: a control buffer that holds the credential but not + * the SCM_RIGHTS. Everything a handler inspects is intact and the message is + * one descriptor short, which is exactly the shape the kernel delivers under + * descriptor pressure. Before the fix both accessors returned NXT_OK here. + */ +static nxt_int_t +nxt_port_ctrunc_test_truncated(nxt_thread_t *thr, nxt_socket_t *pair, + size_t controllen, const char *name) +{ + ssize_t n; + nxt_int_t ret; + nxt_pid_t pid; + nxt_uint_t before, after; + nxt_fd_t fd[2], fd2[2]; + nxt_recv_oob_t oob; + + before = nxt_port_ctrunc_test_fd_count(); + + if (nxt_port_ctrunc_test_exchange(thr, pair, controllen, &oob, &n) + != NXT_OK) + { + return NXT_ERROR; + } + + if (!oob.truncated) { + nxt_log_alert(thr->log, "port ctrunc test: %s did not truncate " + "(controllen %uz, received %uz)", + name, controllen, oob.size); + return NXT_ERROR; + } + + fd[0] = -1; + fd[1] = -1; + + /* + * The two accessors are checked against the same received control block, + * so both see the identical descriptor numbers; they are closed once, at + * the end, rather than after each call. + */ + ret = nxt_socket_msg_oob_get_fds(&oob, fd); + + if (ret != NXT_ERROR) { + nxt_log_alert(thr->log, "port ctrunc test: %s: oob_get_fds accepted " + "a truncated control block (fd %d)", name, fd[0]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + fd2[0] = -1; + fd2[1] = -1; + pid = -1; + + ret = nxt_socket_msg_oob_get(&oob, fd2, &pid); + + if (ret != NXT_ERROR) { + nxt_log_alert(thr->log, "port ctrunc test: %s: oob_get accepted a " + "truncated control block (fd %d, pid %PI)", + name, fd2[0], pid); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + if (fd2[0] != fd[0] || fd2[1] != fd[1]) { + nxt_log_alert(thr->log, "port ctrunc test: %s: the two accessors " + "reported different descriptors (%d %d vs %d %d)", + name, fd[0], fd[1], fd2[0], fd2[1]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + /* + * Whatever the kernel did install has to come back in fd[] even though + * the call failed, or nothing can close it. + */ + nxt_port_ctrunc_test_close_fds(fd); + + after = nxt_port_ctrunc_test_fd_count(); + + if (after != before) { + nxt_log_alert(thr->log, "port ctrunc test: %s leaked %d descriptors", + name, (int) (after - before)); + return NXT_ERROR; + } + + return NXT_OK; +} + +#endif + + +/* + * The control case: a buffer of the size every production receive path uses. + * A message Unit can send never truncates against it, so the accessors accept + * it and hand back both descriptors -- which is what makes the case above an + * assertion about truncation rather than about the buffer being short. + */ +static nxt_int_t +nxt_port_ctrunc_test_intact(nxt_thread_t *thr, nxt_socket_t *pair) +{ + ssize_t n; + nxt_int_t ret; + nxt_pid_t pid; + nxt_uint_t before, after; + nxt_fd_t fd[2]; + nxt_recv_oob_t oob; + + before = nxt_port_ctrunc_test_fd_count(); + + if (nxt_port_ctrunc_test_exchange(thr, pair, NXT_OOB_RECV_SIZE, &oob, &n) + != NXT_OK) + { + return NXT_ERROR; + } + + if (oob.truncated) { + nxt_log_alert(thr->log, "port ctrunc test: a %uz byte control buffer " + "truncated a two descriptor message", + (size_t) NXT_OOB_RECV_SIZE); + return NXT_ERROR; + } + + fd[0] = -1; + fd[1] = -1; + pid = -1; + + ret = nxt_socket_msg_oob_get(&oob, fd, &pid); + + if (ret != NXT_OK || fd[0] == -1 || fd[1] == -1) { + nxt_log_alert(thr->log, "port ctrunc test: intact message refused " + "(ret %i, fd %d %d)", ret, fd[0], fd[1]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + +#if (NXT_CRED_USECMSG) + if (pid != getpid()) { + nxt_log_alert(thr->log, "port ctrunc test: intact message carried " + "pid %PI, expected %PI", pid, (nxt_pid_t) getpid()); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } +#endif + + nxt_port_ctrunc_test_close_fds(fd); + + after = nxt_port_ctrunc_test_fd_count(); + + if (after != before) { + nxt_log_alert(thr->log, "port ctrunc test: intact case leaked %d " + "descriptors", (int) (after - before)); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * The accessor contract on its own, with no kernel behaviour in it: take a + * control block the kernel delivered whole, mark it truncated, and require + * the same refusal. This is what holds everywhere -- the cases above depend + * on how a particular kernel divides a short control buffer, and xnu does not + * divide it the same way -- and it is also the exact shape the accessors see + * in production, since a real truncation can deliver every descriptor it was + * asked for and still raise MSG_CTRUNC. + */ +static nxt_int_t +nxt_port_ctrunc_test_marked(nxt_thread_t *thr, nxt_socket_t *pair) +{ + ssize_t n; + nxt_int_t ret; + nxt_pid_t pid; + nxt_uint_t before, after; + nxt_fd_t fd[2]; + nxt_recv_oob_t oob; + + before = nxt_port_ctrunc_test_fd_count(); + + if (nxt_port_ctrunc_test_exchange(thr, pair, NXT_OOB_RECV_SIZE, &oob, &n) + != NXT_OK) + { + return NXT_ERROR; + } + + if (oob.truncated) { + nxt_log_alert(thr->log, "port ctrunc test: a full size control " + "buffer truncated a two descriptor message"); + return NXT_ERROR; + } + + oob.truncated = 1; + + fd[0] = -1; + fd[1] = -1; + pid = -1; + + ret = nxt_socket_msg_oob_get(&oob, fd, &pid); + + if (ret != NXT_ERROR) { + nxt_log_alert(thr->log, "port ctrunc test: oob_get accepted a block " + "marked truncated (fd %d %d)", fd[0], fd[1]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + /* + * Every descriptor the block carries still has to be reported, or the + * caller that is about to drop the message cannot close them. + */ + if (fd[0] == -1 || fd[1] == -1) { + nxt_log_alert(thr->log, "port ctrunc test: a block marked truncated " + "withheld its descriptors (fd %d %d)", fd[0], fd[1]); + + nxt_port_ctrunc_test_close_fds(fd); + + return NXT_ERROR; + } + + nxt_port_ctrunc_test_close_fds(fd); + + after = nxt_port_ctrunc_test_fd_count(); + + if (after != before) { + nxt_log_alert(thr->log, "port ctrunc test: marked case leaked %d " + "descriptors", (int) (after - before)); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Linux only, deliberately narrower than NXT_CRED_USECMSG. This fixture + * manufactures a message the receiver must refuse by relying on the + * credential being *enforced but not attached*: nxt_socket_msg_oob_get() + * fails a control block that carries no credential, and with SO_PASSCRED off + * this socketpair the kernel adds none. Where the credential travels as + * SCM_CREDS instead (FreeBSD), nxt_socket_msg_oob_init() attaches one itself, + * so the same call produces a perfectly valid message and the refusal this + * fixture is built on never happens. The truncation cases above cover those + * platforms; this one needs Linux semantics to construct its input at all. + */ + +#if (NXT_HAVE_UCRED) + +static nxt_uint_t nxt_port_ctrunc_test_delivered; + + +static void +nxt_port_ctrunc_test_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_ctrunc_test_delivered++; +} + + +static const nxt_port_handlers_t nxt_port_ctrunc_test_handlers = { + .quit = nxt_port_ctrunc_test_quit_handler, +}; + + +static void +nxt_port_ctrunc_test_enable_read_stub(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + /* The fixture engine polls nothing. */ +} + + +/* + * Send one message over "fd" with a control block the receiver will refuse. + * "with_fd" attaches a descriptor, so that the refusal happens after the + * descriptor has been delivered -- the shape a truncation has. + */ +static nxt_int_t +nxt_port_ctrunc_test_send_bad(nxt_thread_t *thr, nxt_socket_t fd, + nxt_port_msg_t *port_msg) +{ + int fds[2]; + ssize_t n; + nxt_send_oob_t oob; + struct iovec iov[1]; + + fds[0] = open("/dev/null", O_RDONLY); + fds[1] = -1; + + if (fds[0] == -1) { + nxt_log_alert(thr->log, "port ctrunc test failed to open /dev/null"); + return NXT_ERROR; + } + + nxt_socket_msg_oob_init(&oob, fds); + + iov[0].iov_base = port_msg; + iov[0].iov_len = sizeof(nxt_port_msg_t); + + n = nxt_sendmsg(fd, iov, 1, &oob); + + (void) close(fds[0]); + + if (n != (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_log_alert(thr->log, "port ctrunc test sendmsg failed %E", + nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * The control message: no control data at all, which nxt_socket_msg_oob_get() + * accepts on any platform, so the handler dispatches it. + */ +static nxt_int_t +nxt_port_ctrunc_test_send_good(nxt_thread_t *thr, nxt_socket_t fd, + nxt_port_msg_t *port_msg) +{ + ssize_t n; + struct iovec iov[1]; + + iov[0].iov_base = port_msg; + iov[0].iov_len = sizeof(nxt_port_msg_t); + + n = nxt_sendmsg(fd, iov, 1, NULL); + + if (n != (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_log_alert(thr->log, "port ctrunc test sendmsg failed %E", + nxt_errno); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Announce a socket message on the queue the way nxt_port_socket_send() does. + * Without the READ_SOCKET marker the handler has port->from_socket == 0 and + * suspends the socket message instead of processing it, which is a different + * path from the one under test. + */ +static nxt_int_t +nxt_port_ctrunc_test_notify(nxt_thread_t *thr, nxt_port_queue_t *queue) +{ + int notify; + uint8_t qmsg; + + qmsg = _NXT_PORT_MSG_READ_SOCKET; + + if (nxt_port_queue_send(queue, &qmsg, 1, ¬ify) != NXT_OK) { + nxt_log_alert(thr->log, "port ctrunc test failed to queue a " + "read_socket marker"); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Put a whole message on the queue, the way nxt_port_socket_write2() does for + * an enqueueable one, and report what it decided about the wakeup. "notify" + * is the crux of the stranding case: nxt_port_queue_send() clears it when + * nitems is already non-zero, on the understanding that a reader is awake and + * will come back round for this message. + */ +static nxt_int_t +nxt_port_ctrunc_test_enqueue(nxt_thread_t *thr, nxt_port_queue_t *queue, + nxt_port_msg_t *port_msg, int *notify) +{ + if (nxt_port_queue_send(queue, port_msg, sizeof(nxt_port_msg_t), notify) + != NXT_OK) + { + nxt_log_alert(thr->log, "port ctrunc test failed to enqueue a " + "queue message"); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * The queue-backed read path drops a refused message rather than failing the + * port, and has to give back everything the iteration took before it does. + * + * queue->nitems is the one that bites: it is incremented on entry and read by + * nxt_port_queue_send() in every *other* process holding this port, which + * skips the socket notification while it is non-zero because a reader is + * supposed to be awake. A bare return therefore left the counter raised + * forever, and the port was never woken for a queue-only message again -- + * one refused datagram costing the port every message after it. The buffer + * and the READ_SOCKET marker leaked on the same path. + * + * The refusal is driven by a control block the accessor rejects for a reason + * that needs no kernel cooperation: SO_PASSCRED is left off this socketpair, + * so the SCM_CREDENTIALS that nxt_socket_msg_oob_get() insists on never + * arrives, and the message is refused after its descriptor was delivered -- + * the same shape, and the same cleanup path, as a truncation. Driving an + * actual MSG_CTRUNC here would mean exhausting the test process's descriptor + * table, which is neither deterministic nor confined to this test. + */ +static nxt_int_t +nxt_port_ctrunc_test_queue(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + int notify; + nxt_int_t ret; + nxt_port_t *port; + nxt_task_t *task; + nxt_runtime_t *rt, *saved_rt; + nxt_port_msg_t port_msg; + nxt_port_queue_t *queue; + nxt_event_engine_t *engine, *saved_engine; + nxt_nncq_atomic_t nitems; + + task = thr->task; + task->thread = thr; + + port = NULL; + queue = MAP_FAILED; + ret = NXT_ERROR; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + engine = nxt_mp_zalloc(mp, sizeof(nxt_event_engine_t)); + + if (nxt_slow_path(rt == NULL || engine == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + thr->runtime = rt; + thr->engine = engine; + + engine->event.enable_read = nxt_port_ctrunc_test_enable_read_stub; + + port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + /* + * Plain socketpair(), not nxt_socketpair_create(): the missing + * SO_PASSCRED is what makes the receiver refuse the message. The + * non-blocking mode that nxt_socketpair_create() also sets is not + * optional, though -- the read loop ends by receiving EAGAIN, so a + * blocking descriptor parks the handler in recvmsg() for good. + */ + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, port->pair) != 0)) { + nxt_log_alert(thr->log, "port ctrunc test: socketpair failed %E", + nxt_errno); + goto done; + } + + if (nxt_slow_path(nxt_socket_nonblocking(task, port->pair[0]) != NXT_OK + || nxt_socket_nonblocking(task, port->pair[1]) != NXT_OK)) + { + goto done; + } + + port->max_size = 1024; + + /* + * nxt_socketpair_recv() logs through ev->task; the event facility would + * have set it when the port was created against a real engine. + */ + port->socket.task = task; + + queue = mmap(NULL, sizeof(nxt_port_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANON, -1, 0); + + if (nxt_slow_path(queue == MAP_FAILED)) { + nxt_log_alert(thr->log, "port ctrunc test failed to map a queue"); + goto done; + } + + nxt_port_queue_init(queue); + + port->queue = queue; + + nxt_port_enable(task, port, &nxt_port_ctrunc_test_handlers); + nxt_port_read_enable(task, port); + + if (nxt_slow_path(port->socket.read_handler == NULL)) { + nxt_log_alert(thr->log, "port ctrunc test: no queue read handler"); + goto done; + } + + nitems = queue->nitems; + + nxt_memzero(&port_msg, sizeof(nxt_port_msg_t)); + port_msg.type = _NXT_PORT_MSG_QUIT; + port_msg.last = 1; + + if (nxt_slow_path(nxt_port_ctrunc_test_send_bad(thr, port->pair[1], + &port_msg) != NXT_OK)) + { + goto done; + } + + if (nxt_slow_path(nxt_port_ctrunc_test_notify(thr, queue) != NXT_OK)) { + goto done; + } + + /* What the event facility would have set. */ + port->socket.read_ready = 1; + + nxt_port_ctrunc_test_delivered = 0; + + port->socket.read_handler(task, &port->socket, NULL); + + if (nxt_port_ctrunc_test_delivered != 0) { + nxt_log_alert(thr->log, "port ctrunc test: a message with an " + "unusable control block was delivered"); + goto done; + } + + if (queue->nitems != nitems) { + nxt_log_alert(thr->log, "port ctrunc test: a refused message left " + "queue->nitems at %uD, was %uD; the port would never " + "be notified again", + (uint32_t) queue->nitems, (uint32_t) nitems); + goto done; + } + + if (port->from_socket != 0) { + nxt_log_alert(thr->log, "port ctrunc test: a refused message left " + "from_socket at %d", (int) port->from_socket); + goto done; + } + + /* + * The point of the counter: the very next message still has to arrive. + */ + if (nxt_slow_path(nxt_port_ctrunc_test_send_good(thr, port->pair[1], + &port_msg) != NXT_OK)) + { + goto done; + } + + if (nxt_slow_path(nxt_port_ctrunc_test_notify(thr, queue) != NXT_OK)) { + goto done; + } + + port->socket.read_ready = 1; + + port->socket.read_handler(task, &port->socket, NULL); + + if (nxt_port_ctrunc_test_delivered != 1) { + nxt_log_alert(thr->log, "port ctrunc test: the message after a " + "refused one was not delivered"); + goto done; + } + + /* + * The case the counter alone does not cover: a message queued inside the + * window, after the READ_SOCKET marker but before this handler gets to + * the datagram it announces. + * + * Its sender sees nitems already raised and so asks for no wakeup -- the + * assertion below -- because a reader is supposed to be awake and about to + * drain the queue. That reader is this invocation. If the refusal path + * returns instead of carrying on round the loop, nothing is left to + * collect the message and nothing will be scheduled to: it is stranded + * with the counter correct and the port quiet. Restoring nitems is + * necessary and not sufficient. + */ + + nxt_port_ctrunc_test_delivered = 0; + + if (nxt_slow_path(nxt_port_ctrunc_test_send_bad(thr, port->pair[1], + &port_msg) != NXT_OK)) + { + goto done; + } + + if (nxt_slow_path(nxt_port_ctrunc_test_notify(thr, queue) != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(nxt_port_ctrunc_test_enqueue(thr, queue, &port_msg, + ¬ify) != NXT_OK)) + { + goto done; + } + + if (notify != 0) { + nxt_log_alert(thr->log, "port ctrunc test: the queued message asked " + "for a wakeup, so it is not the stranding case"); + goto done; + } + + port->socket.read_ready = 1; + + port->socket.read_handler(task, &port->socket, NULL); + + if (nxt_port_ctrunc_test_delivered != 1) { + nxt_log_alert(thr->log, "port ctrunc test: a message queued behind a " + "refused one was stranded"); + goto done; + } + + if (queue->nitems != nitems) { + nxt_log_alert(thr->log, "port ctrunc test: draining past a refused " + "message left queue->nitems at %uD, was %uD", + (uint32_t) queue->nitems, (uint32_t) nitems); + goto done; + } + + ret = NXT_OK; + +done: + + /* + * The port carries its own memory pool and a use count, and its pool + * cleanup asserts that both descriptors are already gone -- so it has to + * be taken down through nxt_port_close(), not with close() on the pair, + * and then released by dropping the reference nxt_port_new() took. The + * queue is detached first: it is this fixture's mapping, unmapped below, + * and nxt_port_close() would otherwise release it as its own. + */ + if (port != NULL) { + port->queue = NULL; + + nxt_port_close(task, port); + nxt_port_use(task, port, -1); + } + + if (queue != MAP_FAILED) { + (void) munmap(queue, sizeof(nxt_port_queue_t)); + } + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} + +#endif /* NXT_HAVE_UCRED */ + + +nxt_int_t +nxt_port_ctrunc_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_socket_t pair[2]; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test started"); + + task = thr->task; + task->thread = thr; + + /* + * nxt_socketpair_create() rather than socketpair(): SO_PASSCRED is what + * puts a credential cmsg ahead of the SCM_RIGHTS, and so what lets a + * buffer sized for the credential alone reproduce the production shape -- + * a fully authenticated message with its descriptor missing. + */ + if (nxt_socketpair_create(task, pair) != NXT_OK) { + nxt_log_alert(thr->log, "port ctrunc test: socketpair failed"); + return NXT_ERROR; + } + + ret = nxt_port_ctrunc_test_intact(thr, pair); + + if (ret == NXT_OK) { + ret = nxt_port_ctrunc_test_marked(thr, pair); + } + + /* + * The cases below assert how the kernel itself divides a control buffer + * that is too small: Linux and the BSDs run scm_detach_fds() against the + * room that is left and raise MSG_CTRUNC, while xnu externalizes the + * rights before it measures, so the same short receive there does not + * report truncation at all. Skipped rather than adapted: what they add + * over nxt_port_ctrunc_test_marked() is the proof that a real kernel + * produces the flag, and on a kernel that does not, there is nothing to + * prove. The code under test is platform independent -- nothing acts on + * a truncation that was not reported. + */ +#if (NXT_MACOSX) + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test: kernel driven " + "truncation cases skipped on macOS"); + +#else + + if (ret == NXT_OK) { + ret = nxt_port_ctrunc_test_truncated(thr, pair, + NXT_CTRUNC_TEST_CRED_SPACE + CMSG_LEN(0), + "credential kept, SCM_RIGHTS dropped"); + } + + if (ret == NXT_OK) { + /* + * The other side of the same kernel decision: room for one descriptor + * where two were sent. scm_detach_fds() installs what fits and still + * raises MSG_CTRUNC, so the accessors have to report the partial + * descriptor and refuse the message, or it leaks. + * + * The length is CMSG_LEN(0) + sizeof(int), not CMSG_SPACE(sizeof(int)): + * cmsg padding rounds a one-descriptor payload up to the same space + * two occupy, so a CMSG_SPACE-derived budget delivers both and does + * not truncate at all. + */ + ret = nxt_port_ctrunc_test_truncated(thr, pair, + NXT_CTRUNC_TEST_CRED_SPACE + + CMSG_LEN(0) + sizeof(int), + "one of two descriptors delivered"); + } + +#endif + + nxt_socket_close(task, pair[0]); + nxt_socket_close(task, pair[1]); + +#if (NXT_HAVE_UCRED) + + if (ret == NXT_OK) { + ret = nxt_port_ctrunc_test_queue(thr); + } + +#else + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test: queue read " + "path case skipped without SCM_CREDENTIALS"); + +#endif + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 35b14e5ef..52c4d1e22 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -237,6 +237,10 @@ main(int argc, char **argv) return 1; } + if (nxt_port_ctrunc_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_fd_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index cba76a724..9af54820c 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -83,6 +83,7 @@ nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); +nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From f491e004c6248e9c946a66b5887801275f721a94 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 02:34:18 +0200 Subject: [PATCH 112/391] router: add an optional bound on an application start An application process is only ever announced by the process itself: nxt_unit_init() sends PROCESS_READY (src/nxt_unit.c:1056), and that is the reply the router's START_PROCESS RPC waits for. A worker that is forked successfully but never gets there answers nothing -- a "type": "external" binary execs the user's image from nxt_app_setup() (src/nxt_application.c:1074) before any Unit code runs, and /bin/sleep is enough -- and, because it stays alive, no REMOVE_PID converts the wait into an error either. Nothing bounded that wait, and there was no way to ask for one. With the default "processes" the consequences reach past the application. nxt_router_conf_apply() (src/nxt_router.c:1310) hands the first application that needs a process to nxt_router_app_rpc_create() and returns; that RPC is its only continuation, so nxt_router_conf_ready() -- the only writer of RPC_READY_LAST to the controller -- is never reached. The controller parks the request at the head of its queue (src/nxt_controller.c:1542) and nxt_controller_check_postpone_request() (src/nxt_controller.c:1669) queues every later control request behind it, GET /status included. The whole control plane is unavailable for as long as that one process lives. On the on-demand path the leak is app->pending_processes, which pins nxt_router_app_can_start() (src/nxt_router.h) false for good. Add "limits": {"start_timeout"}, in seconds like every other "limits" member. A deadline is armed with the START_PROCESS write (src/nxt_router.c:528 on demand, :3575 at configuration apply) and cancelled in whichever reply handler answers first. On expiry the RPC is failed through the new nxt_port_rpc_error() (src/nxt_port_rpc.c:584), which synthesizes the RPC_ERROR that nxt_port_rpc_close() already synthesizes for a whole port, and so runs the handler a real failure would have run: the pending_processes and proto_port_requests accounting stays in nxt_router_app_port_error() (src/nxt_router.c:5291) and nxt_router_app_prefork_error() (src/nxt_router.c:3655) and runs once. The configuration PUT then fails with "Failed to apply new configuration.", the previous configuration stays live, and /status answers again. Expiry cannot fail the RPC on the spot, though, because the START_PROCESS it is giving up on may not have left the router yet. nxt_port_socket_write2() reports NXT_OK once nxt_port_msg_chk_insert() (src/nxt_port_socket.c:334) has put a shallow copy in dport->messages, and that copy holds the payload pointer -- from tmcf->mem_pool on the config-apply path -- and the application's borrowed shared-port descriptor numbers. Failing the RPC runs nxt_router_conf_error(), which unlinks the application (closing that shared port) and releases the pool, and the queued copy would then be dereferenced, and its descriptors sent with SCM_RIGHTS, when the destination finally drained. Before this option, config failure could not run while its own START_PROCESS was still queued: main had to receive the message before it could reject it. So the deadline is armed at the write, and expiry resolves against the send queue first, through the new nxt_port_socket_cancel() (src/nxt_port_socket.c). A message that has not started going out is taken back under the port's write_mutex, its buffers completed and its queue reference discharged outside it, and the descriptors are left alone, because close_fd says they are borrowed. A message that has started (port_msg.nf, set by nxt_port_write_handler() once a fragment and the descriptors are with the peer) is left where it is. Either way the RPC is failed from nxt_router_start_buf_completion(), the one point at which nothing in the port layer can reach the payload any more; only a start whose completion has already run (->send_done) is failed inline. Waiting for the completion is not the same as waiting for the queue to empty: nxt_port_write_handler() removes a fully sent message and only then queues its buffer completion, which can land behind a timer handler the same poll cycle queued ahead of it, so "not in port->messages" is not a lifetime boundary. Both shapes of the message are recalled, not just the one with a payload. A worker start, sent to a prototype that is already up, is header-only: no router memory and no descriptors ride it, so it is no lifetime hazard and the deadline fails its RPC itself, there being no completion to wait on. But delivered after the stream is retired it still starts a worker whose PROCESS_READY answers nobody -- nxt_port_rpc_handler() drops an unknown stream -- leaving a process and a port the router does not track. That is the same orphan a deadline already leaves when a worker is merely slow, and reaping in the prototype is what collects both; recalling the message simply avoids creating one while it has not left yet. Where the destination's port has a shared-memory queue this cannot apply, because the message is already in the ring and only a READ_QUEUE notification can be queued behind it -- which the match on message type leaves alone. Waiting on the completion is only a guarantee if every accepted write delivers one, and the direct-write path did not. When nxt_port_msg_chk_insert() declines to queue, nxt_port_write_handler() sends inline, and a send that fails there -- or an nxt_port_msg_insert_tail() that cannot allocate after an NXT_AGAIN, on a port that is otherwise perfectly alive -- released the caller's message without completing its buffers, while nxt_port_socket_write2() still answered NXT_OK, because the handler returns void. Those buffers were orphaned, and a deadline waiting on one would have waited for good, which is the wedge this option exists to bound. nxt_port_msg_drop() (src/nxt_port_socket.c) completes them, the way nxt_port_error_handler() does for the messages it drains; it cannot double-complete, because a message on these paths was never in port->messages for that drain to find. An expiry is also kept out of the [error] line nxt_router_app_port_error() logs for a start that yielded no process: here one very probably exists -- the router never learns its pid, so it cannot say -- and it simply never called nxt_unit_init(). The deadline's own alert has already said that, naming the application and the knob. The partial-send case is the limitation of a protocol with no cancellation: a destination that accepts one fragment and then stops draining holds the failure off until it drains, and an alert says so. The wedge this option exists for is unaffected -- there the destination is draining fine and it is the worker that never answers -- and the C test pins both. The value is bounded where it is declared: nxt_conf_vldt_app_limits() (src/nxt_conf_validation.c) rejects a negative "start_timeout" and one above NXT_INT32_T_MAX / 1000, the range "idle_timeout" is already held to in nxt_conf_vldt_processes(). NXT_CONF_MAP_MSEC computes (nxt_msec_t) seconds * 1000 and checks nothing of its own, so a larger value lands past the sign bit of the 32-bit millisecond clock, where nxt_msec_diff() (src/nxt_time.h:103) reads the deadline as already past: a bound asked for in days would have fired at once instead. The default is 0, meaning unbounded, so behaviour is unchanged unless the option is set (NXT_APP_START_TIMEOUT, src/nxt_router.h:122). The window the option measures is the whole of the application's own startup, not just Unit's part of it: every module runs user code before nxt_unit_init() -- nxt_python_start() imports the application module (src/python/nxt_python.c:303) before nxt_unit_init() (:365), nxt_java_start() deploys the webapp (src/nxt_java.c:417) before :441, the Node module loads the script before src/nodejs/unit-http/unit.cpp:279, and Go and "external" run main() first -- so a default bound would fail a Django or ML import, or a JVM webapp deployment, that works today. Setting it too low has a cost of its own, and it is documented rather than hidden: the router cannot kill a process it has no pid for, so a worker that was merely slow keeps running, and its late PROCESS_READY creates a port and maps a queue for a stream nxt_port_rpc_handler() has already retired. That worker is orphaned until the application's configuration is replaced. Reaping it belongs to the prototype, which does know the pid, in nxt_proto_sigchld_handler()/nxt_process_quit(); that is the code #268 rewrites, so it is deliberately left out here and pinned by test_app_start_timeout_worker_not_reaped(). The start state is reference counted, because the three things that can end it -- the RPC retiring, the timer node going away, and the buffer completion running -- have no fixed order. The engine's reference is still handed back through the re-arm-at-zero idiom nxt_router_free_app() uses for app_joint->idle_timer (src/nxt_router.c), because nxt_timer_delete()'s removal can itself be a queued change referencing the timer, and a ->firing flag keeps that reference with the expiry handler when the error handler it drives reaches the cancel path. It is created before the write, so the payload can carry it as its completion context, and it holds no application reference, only a copy of the name, so a deadline cannot extend the lifetime of the start it is giving up on. The payload's completion handler is per call site rather than shared: the config-apply buffer is freed with tmcf->mem_pool and had nxt_buf_dummy_completion(), while the on-demand buffer owns itself in the engine's pool and had nxt_buf_completion(), which asserts on b->parent and would drive parent completion for the context stored there. A timeout that fails the request does not stop the worker: the router only learns a worker's pid from PROCESS_READY, which by definition never arrives for a start that expired. The child kept running, and because the expiry released the pending slot while the process count had never been incremented, it was counted nowhere at all. processes.max then bounded nothing: five ordinary client requests against a wedged application produced five surviving children with "max": 2, and they outlived unitd itself. Without this option the first attempt wedges pending_processes and can_start() goes false, so exactly one child leaks -- the wedge was acting as the bound, and removing it removed that too. Count those children in app->unaccounted_processes and include the term in nxt_router_app_can_start(), so max bounds the processes an application may leave behind, whatever became of them. Past the bound a start fails immediately instead of forking another orphan, and the control plane stays up. The slot is not stranded there. The router attributes a start by stream rather than by pid, and nxt_port_remove_notify_others() sends REMOVE_PID with the process's stream attached, so re-registering a handler on that stream after the deadline keeps both answers reachable: a worker that finally dies drains the slot through the RPC error, and a merely slow worker that announces itself is adopted into ->processes instead of being discarded -- discarding it would fork an equally slow replacement and fail it on the same deadline. A start recalled from the send queue before the prototype read it forked nothing, and is deliberately not counted; accounting it would move a slot to a counter nothing can drain and turn the bound into a ratchet. Neither drain is guaranteed. A worker wedged forever holds its slot for the life of the application, and after max of those it refuses to start more until a reconfiguration replaces it. That is the trade: bounded processes and fast failures instead of an unbounded leak, with the wedge confined to the application that caused it. "GET /status" reports the count as "unaccounted" beside "starting". A prototype start expires the same way, and leaves the same kind of process behind: the prototype forks no worker of its own, but it is itself a forked process, and the accounting excluded it. Because the handler then clears proto_port_requests and returns the pending slots, every later request could fork another stuck prototype -- six of them at "max": 2 in a measurement, one more per request, none of them counted. Reaping no longer keys on whether the start was for a prototype: its cohort stands for requests, so the surplus slots go back the ordinary way and one is accounted, drained by the same REMOVE_PID or late PROCESS_READY as a worker's. Clearing the cohort also lets a second prototype start while the first is still on its way, so two can become ready at once; the assertion that this could not happen is replaced by disposing of the superseded one, which a release build would otherwise have overwritten and leaked. A start the deadline could not take back whole -- partly sent, or dropped from the queue -- forked nothing either, and is excluded like a recalled one; widening the reap without that would have turned those into a ratchet. Fixes #275. Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + docs/unit-openapi.yaml | 52 ++ src/nxt_conf_validation.c | 73 +- src/nxt_port.h | 43 + src/nxt_port_rpc.c | 110 ++- src/nxt_port_rpc.h | 4 + src/nxt_port_socket.c | 164 +++- src/nxt_router.c | 1089 +++++++++++++++++++++- src/nxt_router.h | 58 +- src/nxt_status.c | 7 +- src/nxt_status.h | 1 + src/test/nxt_router_start_timeout_test.c | 1023 ++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/test_app_start_timeout.py | 570 +++++++++++ test/test_status.py | 1 + 16 files changed, 3120 insertions(+), 81 deletions(-) create mode 100644 src/test/nxt_router_start_timeout_test.c create mode 100644 test/test_app_start_timeout.py diff --git a/auto/sources b/auto/sources index 32297b797..3c1fd2d6d 100644 --- a/auto/sources +++ b/auto/sources @@ -184,6 +184,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_start_fail_soak_test.c \ src/test/nxt_router_proto_wedge_test.c \ src/test/nxt_router_proto_death_test.c \ + src/test/nxt_router_start_timeout_test.c \ src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 3c0d5a32c..a81d78778 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -5445,6 +5445,39 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /status/applications/{appName}/processes/unaccounted: + summary: "Endpoint for the `unaccounted` processes number" + get: + operationId: getStatusApplicationsAppProcessesUnaccounted + summary: "Retrieve the unaccounted processes app status number" + description: "Retrieves the `unaccounted` processes number that + represents Unit's per-app + [process statistics](https://unit.nginx.org/statusapi/)." + + tags: + - status + + parameters: + - $ref: "#/components/parameters/appName" + + responses: + "200": + description: "OK; the `unaccounted` number exists in the + configuration." + + content: + application/json: + schema: + type: integer + format: int64 + + examples: + Unaccounted: + value: 0 + + "404": + $ref: "#/components/responses/responseNotFound" + /status/applications/{appName}/processes/idle: summary: "Endpoint for the `idle` processes number" get: @@ -6309,6 +6342,7 @@ components: processes: running: 9 starting: 1 + unaccounted: 0 idle: 0 requests: active: 15 @@ -6369,6 +6403,7 @@ components: processes: running: 9 starting: 1 + unaccounted: 0 idle: 0 requests: active: 15 @@ -6380,6 +6415,7 @@ components: processes: running: 9 starting: 1 + unaccounted: 0 idle: 0 requests: active: 15 @@ -6390,6 +6426,7 @@ components: value: running: 9 starting: 1 + unaccounted: 0 idle: 0 # /status/applications/{appName}/requests @@ -6903,6 +6940,12 @@ components: type: integer description: "Request timeout in seconds." + start_timeout: + type: integer + description: "Time in seconds an application process may take to + announce itself before Unit fails the start request; 0, the + default, means no limit." + processes: description: "Governs the behavior of app processes." anyOf: @@ -7923,6 +7966,15 @@ components: type: integer description: "Current starting app processes." + unaccounted: + type: integer + format: int64 + description: "App processes Unit started and can no longer see: a + worker that a `limits/start_timeout` deadline gave up on is still + running, but never announced itself, so Unit has no pid for it. + They count against `processes/max` until they announce themselves + after all, or exit." + idle: type: integer description: "Current idle app processes." diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 8d50c3deb..5538b1a7e 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -195,6 +195,8 @@ static nxt_int_t nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value); static nxt_int_t nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_app_limits(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_processes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_object_iterator(nxt_conf_validation_t *vldt, @@ -1306,7 +1308,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { }, { .name = nxt_string("limits"), .type = NXT_CONF_VLDT_OBJECT, - .validator = nxt_conf_vldt_object, + .validator = nxt_conf_vldt_app_limits, .u.members = nxt_conf_vldt_app_limits_members, }, { .name = nxt_string("processes"), @@ -1358,6 +1360,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { { .name = nxt_string("timeout"), .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("start_timeout"), + .type = NXT_CONF_VLDT_INTEGER, }, { .name = nxt_string("requests"), .type = NXT_CONF_VLDT_INTEGER, @@ -3608,6 +3613,72 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +typedef struct { + int64_t start_timeout; +} nxt_conf_vldt_app_limits_conf_t; + + +static nxt_conf_map_t nxt_conf_vldt_app_limits_conf_map[] = { + { + nxt_string("start_timeout"), + NXT_CONF_MAP_INT64, + offsetof(nxt_conf_vldt_app_limits_conf_t, start_timeout), + }, +}; + + +/* + * "start_timeout" reaches the router through NXT_CONF_MAP_MSEC + * (nxt_router_app_limits_conf[], src/nxt_router.c:1733), which computes + * (nxt_msec_t) seconds * 1000 and range-checks nothing. Seconds above + * NXT_INT32_T_MAX / 1000 therefore land past the sign bit of the 32-bit + * millisecond clock, where nxt_msec_diff() (src/nxt_time.h:103) reads the + * deadline as already past, so the bound fires at once instead of far in the + * future; a negative value is an out-of-range conversion to an unsigned type + * before it even gets that far. Bound it here, exactly as "idle_timeout" is + * bounded in nxt_conf_vldt_processes() below. + */ + +static nxt_int_t +nxt_conf_vldt_app_limits(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + nxt_int_t ret; + nxt_conf_vldt_app_limits_conf_t limits; + + static const nxt_str_t start_timeout_str = nxt_string("start_timeout"); + + ret = nxt_conf_vldt_object(vldt, value, data); + if (ret != NXT_OK) { + return ret; + } + + limits.start_timeout = 0; + + ret = nxt_conf_map_object(vldt->pool, value, + nxt_conf_vldt_app_limits_conf_map, + nxt_nitems(nxt_conf_vldt_app_limits_conf_map), + &limits); + if (ret != NXT_OK) { + return ret; + } + + if (limits.start_timeout < 0) { + return nxt_conf_vldt_member_error(vldt, &start_timeout_str, + "The \"start_timeout\" number must not " + "be negative."); + } + + if (limits.start_timeout > NXT_INT32_T_MAX / 1000) { + return nxt_conf_vldt_member_error(vldt, &start_timeout_str, + "The \"start_timeout\" number must not " + "exceed %d.", NXT_INT32_T_MAX / 1000); + } + + return NXT_OK; +} + + typedef struct { int64_t spare; int64_t max; diff --git a/src/nxt_port.h b/src/nxt_port.h index 79b05d42f..0d1d1c722 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -403,6 +403,49 @@ nxt_int_t nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, nxt_fd_t fd, nxt_fd_t fd2, uint32_t stream, nxt_port_id_t reply_port, nxt_buf_t *b); + +typedef enum { + /* + * The message was still whole in port->messages and has been taken back + * out of it. Its buffers have been completed and the queue's reference + * to the port released, so nothing the caller handed to write2() is + * reachable from the port any more. + */ + NXT_PORT_MSG_CANCELLED = 0, + /* + * Found, but a fragment of it has already gone out (port_msg.nf), so the + * peer is mid-stream and the descriptors have been handed off. Left + * queued: taking it back now would strand the peer on an unfinished + * stream. + */ + NXT_PORT_MSG_STARTED, + /* + * Not in port->messages. Either it was never queued or it has been sent + * in full. NOT a lifetime boundary on its own: the write handler removes + * the message and only then queues the buffer completion, which can land + * behind work already queued ahead of it. + */ + NXT_PORT_MSG_NOT_FOUND, +} nxt_port_msg_cancel_t; + +/* + * Take a not-yet-started message back out of a port's send queue. + * + * Callable only on the thread that owns the port's engine, and only against a + * message this caller queued: it identifies the message by (type, stream, + * reply_port) and, when b is not NULL, by buffer identity as well, since a + * stream number alone is only unique per reply port. + * + * Descriptors are treated exactly as the send path treats them, through + * close_fd: a message that owns its descriptors has them closed here, and one + * that merely borrows them (START_PROCESS borrows the application's shared + * port) does not, so cancelling can never close a descriptor its owner is + * still using. + */ +nxt_port_msg_cancel_t nxt_port_socket_cancel(nxt_task_t *task, + nxt_port_t *port, nxt_uint_t type, uint32_t stream, + nxt_port_id_t reply_port, nxt_buf_t *b); + #if (NXT_TESTS) void nxt_port_test_msg_alloc_failures(nxt_uint_t failures); void nxt_port_test_run_error_handler(nxt_task_t *task, nxt_port_t *port); diff --git a/src/nxt_port_rpc.c b/src/nxt_port_rpc.c index 87815faae..0cdd796fa 100644 --- a/src/nxt_port_rpc.c +++ b/src/nxt_port_rpc.c @@ -25,6 +25,9 @@ struct nxt_port_rpc_reg_s { }; +static void *nxt_port_rpc_reg_add(nxt_task_t *task, nxt_port_t *port, + nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, + uint32_t stream, size_t ex_size); static void nxt_port_rpc_remove_from_peers(nxt_task_t *task, nxt_port_t *port, nxt_port_rpc_reg_t *reg); @@ -155,14 +158,50 @@ nxt_port_rpc_register_handler_ex(nxt_task_t *task, nxt_port_t *port, nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, size_t ex_size) { - uint32_t stream; + return nxt_port_rpc_reg_add(task, port, ready_handler, error_handler, + nxt_atomic_fetch_add(nxt_stream_ident, 1), + ex_size); +} + + +/* + * Register a handler on a stream the caller names rather than on a fresh one. + * + * The one caller is a router that has just been told, by its own deadline, + * that a start it asked for will not be answered, and that wants to keep + * watching the stream for the reply that may still come: the worker's late + * PROCESS_READY, or the REMOVE_PID for it that the process's death produces. + * See nxt_router_app_start_expired() in src/nxt_router.c. + * + * Only safe from inside that stream's own handler, because + * nxt_port_rpc_handler() deletes the registration before it runs the handler + * and does not delete again afterwards -- so the key is free at that point, + * and nothing removes what this puts back. Anywhere else the insert would + * either collide with a live registration or be undone by the caller's own + * cleanup, and the stream identifier itself is only unique because + * nxt_stream_ident is never reused. + */ + +void * +nxt_port_rpc_register_handler_at(nxt_task_t *task, nxt_port_t *port, + nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, + uint32_t stream, size_t ex_size) +{ + return nxt_port_rpc_reg_add(task, port, ready_handler, error_handler, + stream, ex_size); +} + + +static void * +nxt_port_rpc_reg_add(nxt_task_t *task, nxt_port_t *port, + nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, + uint32_t stream, size_t ex_size) +{ nxt_port_rpc_reg_t *reg; nxt_lvlhsh_query_t lhq; nxt_assert(port->pair[0] != -1); - stream = nxt_atomic_fetch_add(nxt_stream_ident, 1); - #if (NXT_TESTS) if (nxt_slow_path(nxt_port_rpc_test_should_fail( &nxt_port_rpc_test_alloc_failure_count))) @@ -544,11 +583,55 @@ nxt_port_rpc_cancel(nxt_task_t *task, nxt_port_t *port, uint32_t stream) static nxt_buf_t nxt_port_close_dummy_buf; + +static void +nxt_port_rpc_error_msg(nxt_task_t *task, nxt_port_t *port, uint32_t stream) +{ + nxt_port_recv_msg_t msg; + + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.buf = &nxt_port_close_dummy_buf; + msg.port = port; + msg.port_msg.stream = stream; + msg.port_msg.pid = nxt_pid; + msg.port_msg.type = _NXT_PORT_MSG_RPC_ERROR; + msg.port_msg.last = 1; + msg.port_msg.mmap = 0; + msg.port_msg.nf = 0; + msg.port_msg.mf = 0; + msg.size = 0; + msg.cancelled = 0; + msg.u.data = NULL; + + nxt_port_rpc_handler(task, &msg); +} + + +/* + * Fail one registered RPC as though the peer had answered RPC_ERROR: the + * registered error handler runs and the registration is retired, so a caller + * that has decided the answer will never come reuses that handler's recovery + * instead of duplicating it. + * + * Safe on a stream that is already gone -- nxt_port_rpc_handler() drops an + * unknown stream with a debug line -- which is what makes it usable from a + * deadline timer racing a real reply. + */ + +void +nxt_port_rpc_error(nxt_task_t *task, nxt_port_t *port, uint32_t stream) +{ + nxt_debug(task, "rpc: stream #%uD fail registration", stream); + + nxt_port_rpc_error_msg(task, port, stream); +} + + void nxt_port_rpc_close(nxt_task_t *task, nxt_port_t *port) { - nxt_port_rpc_reg_t *reg; - nxt_port_recv_msg_t msg; + nxt_port_rpc_reg_t *reg; for ( ;; ) { reg = nxt_lvlhsh_peek(&port->rpc_streams, &lvlhsh_rpc_reg_proto); @@ -556,21 +639,6 @@ nxt_port_rpc_close(nxt_task_t *task, nxt_port_t *port) return; } - msg.fd[0] = -1; - msg.fd[1] = -1; - msg.buf = &nxt_port_close_dummy_buf; - msg.port = port; - msg.port_msg.stream = reg->stream; - msg.port_msg.pid = nxt_pid; - msg.port_msg.type = _NXT_PORT_MSG_RPC_ERROR; - msg.port_msg.last = 1; - msg.port_msg.mmap = 0; - msg.port_msg.nf = 0; - msg.port_msg.mf = 0; - msg.size = 0; - msg.cancelled = 0; - msg.u.data = NULL; - - nxt_port_rpc_handler(task, &msg); + nxt_port_rpc_error_msg(task, port, reg->stream); } } diff --git a/src/nxt_port_rpc.h b/src/nxt_port_rpc.h index 8218c4e3a..b30140b4e 100644 --- a/src/nxt_port_rpc.h +++ b/src/nxt_port_rpc.h @@ -19,6 +19,9 @@ uint32_t nxt_port_rpc_register_handler(nxt_task_t *task, nxt_port_t *port, void *nxt_port_rpc_register_handler_ex(nxt_task_t *task, nxt_port_t *port, nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, size_t ex_size); +void *nxt_port_rpc_register_handler_at(nxt_task_t *task, nxt_port_t *port, + nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, + uint32_t stream, size_t ex_size); #if (NXT_TESTS) void nxt_port_rpc_test_alloc_failures(nxt_uint_t failures); @@ -33,6 +36,7 @@ void nxt_port_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_port_rpc_remove_peer(nxt_task_t *task, nxt_port_t *port, nxt_pid_t peer); void nxt_port_rpc_cancel(nxt_task_t *task, nxt_port_t *port, uint32_t stream); +void nxt_port_rpc_error(nxt_task_t *task, nxt_port_t *port, uint32_t stream); void nxt_port_rpc_close(nxt_task_t *task, nxt_port_t *port); diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 84dd1b37c..7a4f868fb 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -187,6 +187,52 @@ nxt_port_release_send_msg(nxt_port_send_msg_t *msg) } +/* + * Give up on a message that nxt_port_write_handler() is dropping, completing + * what it still owns. + * + * Only for a message that is not in port->messages -- msg->link.next == NULL, + * which on these paths means it is still the caller's stack copy from the + * nxt_port_msg_chk_insert() NXT_DECLINED branch. That is also why this cannot + * double-complete: the nxt_port_error_handler() the caller goes on to raise + * completes what the queue holds, and this was never in it. + * + * Without this the buffers of an accepted write were simply orphaned, which + * breaks the contract nxt_port_socket_write2() documents -- it answers NXT_OK + * on these paths, because nxt_port_write_handler() returns void -- and strands + * any caller waiting on the completion, such as the START_PROCESS deadline in + * src/nxt_router.c. + * + * Descriptors first, then buffers, the order nxt_port_error_handler() uses. + */ + +static void +nxt_port_msg_drop(nxt_task_t *task, nxt_port_send_msg_t *msg) +{ + nxt_buf_t *b, *next; + nxt_work_queue_t *wq; + + nxt_port_msg_close_fd(msg); + + wq = &task->thread->engine->fast_work_queue; + + for (b = msg->buf; b != NULL; b = next) { + next = b->next; + b->next = NULL; + + if (nxt_buf_is_sync(b)) { + continue; + } + + nxt_work_queue_add(wq, b->completion_handler, task, b, b->parent); + } + + msg->buf = NULL; + + nxt_port_release_send_msg(msg); +} + + nxt_int_t nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, nxt_fd_t fd, nxt_fd_t fd2, uint32_t stream, nxt_port_id_t reply_port, @@ -413,7 +459,7 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) struct iovec iov[NXT_IOBUF_MAX * 10]; nxt_work_queue_t *wq; nxt_port_method_t m; - nxt_port_send_msg_t *msg; + nxt_port_send_msg_t *msg, *qmsg; nxt_sendbuf_coalesce_t sb; port = nxt_container_of(obj, nxt_port_t, socket); @@ -520,11 +566,15 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) nxt_thread_mutex_unlock(&port->write_mutex); } else { - msg = nxt_port_msg_insert_tail(port, msg); - if (nxt_slow_path(msg == NULL)) { + qmsg = nxt_port_msg_insert_tail(port, msg); + if (nxt_slow_path(qmsg == NULL)) { + nxt_port_msg_drop(task, msg); + goto fail; } + msg = qmsg; + use_delta++; } @@ -552,22 +602,32 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) } } else { + /* + * Both failures below drop a message that was never queued, so + * both go through nxt_port_msg_drop(): the send is over and + * nothing else will ever complete its buffers. Neither is a + * socket-death-only path -- the second is an allocation failure + * against a port that is otherwise perfectly alive. + */ + if (nxt_slow_path(n == NXT_ERROR)) { if (msg->link.next == NULL) { - nxt_port_msg_close_fd(msg); - - nxt_port_release_send_msg(msg); + nxt_port_msg_drop(task, msg); } goto fail; } if (msg->link.next == NULL) { - msg = nxt_port_msg_insert_tail(port, msg); - if (nxt_slow_path(msg == NULL)) { + qmsg = nxt_port_msg_insert_tail(port, msg); + if (nxt_slow_path(qmsg == NULL)) { + nxt_port_msg_drop(task, msg); + goto fail; } + msg = qmsg; + use_delta++; } } @@ -603,6 +663,94 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) } +nxt_port_msg_cancel_t +nxt_port_socket_cancel(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, + uint32_t stream, nxt_port_id_t reply_port, nxt_buf_t *b) +{ + nxt_buf_t *buf, *next; + nxt_work_queue_t *wq; + nxt_port_send_msg_t *msg, *found; + nxt_port_msg_cancel_t ret; + + found = NULL; + ret = NXT_PORT_MSG_NOT_FOUND; + + nxt_thread_mutex_lock(&port->write_mutex); + + nxt_queue_each(msg, &port->messages, nxt_port_send_msg_t, link) { + + if (msg->port_msg.stream != stream + || msg->port_msg.type != (type & NXT_PORT_MSG_MASK) + || msg->port_msg.reply_port != reply_port) + { + continue; + } + + if (b != NULL && msg->buf != b) { + continue; + } + + /* + * nxt_port_write_handler() sets nf on the message once it has sent a + * fragment, and clears msg->fd[] at the same time: from then on the + * peer is reading a stream this message must finish. + */ + + if (msg->port_msg.nf) { + ret = NXT_PORT_MSG_STARTED; + break; + } + + nxt_queue_remove(&msg->link); + msg->link.next = NULL; + + found = msg; + ret = NXT_PORT_MSG_CANCELLED; + + break; + + } nxt_queue_loop; + + nxt_thread_mutex_unlock(&port->write_mutex); + + if (found == NULL) { + return ret; + } + + /* + * Outside the mutex from here: a completion handler may reach arbitrary + * router code, and nxt_port_use() can destroy the port. + */ + + buf = found->buf; + found->buf = NULL; + + nxt_port_msg_close_fd(found); + + nxt_port_release_send_msg(found); + + wq = &task->thread->engine->fast_work_queue; + + while (buf != NULL) { + next = buf->next; + buf->next = NULL; + + if (!nxt_buf_is_sync(buf)) { + nxt_work_queue_add(wq, buf->completion_handler, task, buf, + buf->parent); + } + + buf = next; + } + + /* The reference nxt_port_msg_chk_insert() took for the queued message. */ + + nxt_port_use(task, port, -1); + + return ret; +} + + static nxt_port_send_msg_t * nxt_port_msg_first(nxt_port_t *port) { diff --git a/src/nxt_router.c b/src/nxt_router.c index ce1d09564..0517ac432 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -40,6 +40,7 @@ typedef struct { nxt_conf_value_t *limits_value; nxt_conf_value_t *processes_value; nxt_conf_value_t *targets_value; + nxt_msec_t start_timeout; } nxt_router_app_conf_t; @@ -84,20 +85,129 @@ typedef struct { } nxt_socket_rpc_t; +typedef struct nxt_router_start_timer_s nxt_router_start_timer_t; + + typedef struct { - nxt_app_t *app; - nxt_router_temp_conf_t *temp_conf; - uint8_t proto; /* 1 bit */ + nxt_app_t *app; + nxt_router_temp_conf_t *temp_conf; + nxt_router_start_timer_t *start_timer; + uint8_t proto; /* 1 bit */ } nxt_app_rpc_t; typedef struct { - nxt_app_joint_t *app_joint; - uint32_t generation; - uint8_t proto; /* 1 bit */ + nxt_app_joint_t *app_joint; + nxt_router_start_timer_t *start_timer; + uint32_t generation; + uint8_t proto; /* 1 bit */ + /* + * Not a start attempt at all any more: the registration was put back on + * the stream by nxt_router_app_start_expired() to reap the process the + * expired attempt left behind, and it owns an unaccounted_processes slot + * rather than a pending_processes one. + */ + uint8_t expired; /* 1 bit */ } nxt_app_joint_rpc_t; +/* + * Deadline for one START_PROCESS RPC. + * + * The only thing that can answer a start is the new worker itself, through + * nxt_unit_init() -> PROCESS_READY. A process that is forked successfully but + * never gets there -- a "type": "external" binary that blocks before exec'ing + * anything of ours, a runtime stuck in its own init -- answers nothing and + * dies of nothing, so the RPC stays armed for the life of the router. With + * the default "processes" that RPC is the sole continuation of + * nxt_router_conf_apply(), so the configuration PUT never returns and the + * controller queues every later request behind it, GET /status included. + * + * On expiry the RPC is failed through nxt_port_rpc_error(), which runs the + * handler that a real failure would have run. Nothing here duplicates that + * recovery: the pending_processes and proto_port_requests accounting stays in + * nxt_router_app_port_error() / nxt_router_app_prefork_error() and runs once, + * driven by the RPC layer. + * + * The struct owns nothing but a copy of the application name, deliberately: + * holding an nxt_app_t or an nxt_app_joint_t reference would let a deadline + * extend the lifetime of the very object whose start it is giving up on. Both + * ports are referenced, because the stream is only meaningful against the + * router port and the send queue is only reachable through the destination. + * + * Expiry cannot simply fail the RPC, because the START_PROCESS it is giving up + * on may not have left the router yet. nxt_port_socket_write2() reports NXT_OK + * once the message is in dport->messages, and what it queued there is a shallow + * copy: the payload pointer, which on the config-apply path lives in + * tmcf->mem_pool, and the application's shared-port descriptor numbers, which + * it borrows. Failing the RPC runs nxt_router_conf_error(), which unlinks the + * application (closing that shared port) and releases that pool -- and the + * queued copy would still be there, to be dereferenced and sent with + * SCM_RIGHTS when the destination finally drains. + * + * So the deadline is still armed at the write, and expiry resolves against the + * send queue first: + * + * not yet started nxt_port_socket_cancel() takes the message back and + * completes its buffer, and the failure is driven from + * that completion. + * partially sent left queued -- a fragment and the descriptors are + * already with the peer -- and the failure waits for the + * completion of the last fragment. See the note on + * ->expired below. + * fully sent the message is gone from the queue but its completion + * can still be pending: nxt_port_write_handler() removes + * it and only then queues the buffer completion, which + * can land behind a timer handler already queued ahead of + * it. Absence from the queue is not a lifetime boundary, + * so the failure again waits for the completion. + * completed ->send_done: nothing references the payload any more, + * so the RPC is failed inline. + * + * Everything but the last case therefore funnels through + * nxt_router_start_buf_completion(), which is the single point where the + * payload is known to be unreachable from the port layer. + * + * References: one for the RPC that holds it, one for the engine while the + * timer node is live, and one for a buffer completion that has yet to run. + */ + +struct nxt_router_start_timer_s { + nxt_timer_t timer; + /* The port the RPC is registered on, and the one it was written to. */ + nxt_port_t *port; + nxt_port_t *dport; + /* The queued message's identity, for nxt_port_socket_cancel(). */ + nxt_buf_t *buf; + nxt_port_id_t reply_port; + uint32_t stream; + nxt_msec_t timeout; + int32_t refs; + /* The timer node is live in the engine. */ + uint8_t armed; /* 1 bit */ + /* The deadline fired; the RPC must be failed once the payload is free. */ + uint8_t expired; /* 1 bit */ + /* The buffer completion has run: the port layer is done with the msg. */ + uint8_t send_done; /* 1 bit */ + /* The RPC has retired, so nothing may fail it again. */ + uint8_t retired; /* 1 bit */ + /* + * The START_PROCESS was taken back out of the send queue before the + * destination ever saw it, so no process was forked for it and there is + * nothing to account for. See nxt_router_app_port_error(). + */ + uint8_t recalled; /* 1 bit */ + /* + * The payload's completion found bytes of it still unsent, so the + * destination never saw the whole message and forked nothing for it. + * The same conclusion as ->recalled, reached at the other end of a send + * the deadline could not take back. See nxt_router_app_port_error(). + */ + uint8_t dropped; /* 1 bit */ + nxt_str_t app_name; +}; + + static nxt_int_t nxt_router_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp); static nxt_int_t nxt_router_start(nxt_task_t *task, nxt_process_data_t *data); @@ -225,12 +335,35 @@ static void nxt_router_req_headers_ack_handler(nxt_task_t *task, static void nxt_router_listen_socket_release(nxt_task_t *task, nxt_socket_conf_t *skcf); +static nxt_router_start_timer_t *nxt_router_start_timer_create(nxt_task_t *task, + nxt_app_t *app, nxt_port_t *port, nxt_port_t *dport, nxt_buf_t *b); +static void nxt_router_start_timer_arm(nxt_task_t *task, + nxt_router_start_timer_t *st, uint32_t stream); +static void nxt_router_start_timer_cancel(nxt_task_t *task, + nxt_router_start_timer_t *st); +static void nxt_router_start_timer_use(nxt_task_t *task, + nxt_router_start_timer_t *st, int32_t delta); +static void nxt_router_start_timeout(nxt_task_t *task, void *obj, void *data); +static void nxt_router_start_timer_release(nxt_task_t *task, void *obj, + void *data); +static void nxt_router_start_buf_completion(nxt_task_t *task, + nxt_router_start_timer_t *st, nxt_bool_t dropped); +static void nxt_router_start_conf_buf_completion(nxt_task_t *task, void *obj, + void *data); +static void nxt_router_start_app_buf_completion(nxt_task_t *task, void *obj, + void *data); + static void nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static void nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static void nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, - uint32_t count); + uint32_t count, nxt_bool_t unaccounted); +static void nxt_router_app_start_expired(nxt_task_t *task, nxt_app_t *app, + nxt_port_recv_msg_t *msg, nxt_app_joint_t *app_joint, uint32_t generation, + nxt_bool_t proto); +static void nxt_router_app_unaccounted_release(nxt_task_t *task, + nxt_app_t *app); static void nxt_router_app_use(nxt_task_t *task, nxt_app_t *app, int i); static void nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app); @@ -401,18 +534,21 @@ void nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, void *data) { - size_t size; - uint32_t stream; - nxt_fd_t port_fd, queue_fd; - nxt_int_t ret; - nxt_app_t *app; - nxt_buf_t *b; - nxt_port_t *dport; - nxt_runtime_t *rt; - nxt_app_joint_rpc_t *app_joint_rpc; + size_t size; + uint32_t stream; + nxt_fd_t port_fd, queue_fd; + nxt_int_t ret; + nxt_app_t *app; + nxt_buf_t *b; + nxt_port_t *dport; + nxt_runtime_t *rt; + nxt_app_joint_rpc_t *app_joint_rpc; + nxt_router_start_timer_t *st; app = data; + st = NULL; + nxt_thread_mutex_lock(&app->mutex); dport = app->proto_port; @@ -466,6 +602,17 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, stream = nxt_port_rpc_ex_stream(app_joint_rpc); + /* + * Before the write: the deadline resolves through the payload's completion + * handler, which nxt_port_socket_write2() may run before it returns. + */ + + st = nxt_router_start_timer_create(task, app, port, dport, b); + + if (st != NULL && b != NULL) { + b->completion_handler = nxt_router_start_app_buf_completion; + } + ret = nxt_port_socket_write2(task, dport, NXT_PORT_MSG_START_PROCESS, port_fd, queue_fd, stream, port->id, b); if (nxt_slow_path(ret != NXT_OK)) { @@ -477,6 +624,11 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, app_joint_rpc->app_joint = app->joint; app_joint_rpc->generation = app->generation; app_joint_rpc->proto = (b != NULL); + app_joint_rpc->start_timer = st; + + if (st != NULL) { + nxt_router_start_timer_arm(task, st, stream); + } /* * Key the registration by the process the START_PROCESS was sent to, so @@ -520,11 +672,24 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, nxt_alert(task, "app '%V' failed to start a process", &app->name); + if (st != NULL) { + /* + * Never armed, and the buffer below is freed without its completion + * handler ever running, so both references are dropped here. + */ + + if (b != NULL) { + nxt_router_start_timer_use(task, st, -1); + } + + nxt_router_start_timer_use(task, st, -1); + } + if (b != NULL) { nxt_mp_free(b->data, b); } - nxt_router_app_start_failed(task, app, 1); + nxt_router_app_start_failed(task, app, 1, 0); skip: @@ -532,6 +697,418 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, } +/* + * Create the deadline for a START_PROCESS RPC, before the write. + * + * Before, because the payload buffer has to carry a completion handler that + * knows about this struct: the deadline resolves through that completion, and + * nxt_port_socket_write2() can run it before it returns. + * + * The timer runs on the engine of the port the RPC is registered on -- always + * the router's own port, for both the config-apply prefork path + * (nxt_router_app_rpc_create()) and the on-demand path + * (nxt_router_start_app_process_handler(), reached by nxt_port_post() to that + * same port) -- so the expiry handler, the reply handlers it races and the + * buffer completion cannot run concurrently. + * + * A NULL return means no deadline, which is the configured behaviour for + * "start_timeout": 0 and the fallback if the allocation fails; the start is + * then exactly as unbounded as it was before this existed, which is worse than + * the alternative but not worse than failing a start over a small malloc. + */ + +static nxt_router_start_timer_t * +nxt_router_start_timer_create(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port, nxt_port_t *dport, nxt_buf_t *b) +{ + nxt_msec_t timeout; + nxt_router_start_timer_t *st; + + /* Immutable once the application is configured, so read unlocked. */ + timeout = app->start_timeout; + + if (timeout == 0) { + return NULL; + } + + st = nxt_malloc(sizeof(nxt_router_start_timer_t) + app->name.length); + if (nxt_slow_path(st == NULL)) { + nxt_alert(task, "app \"%V\" start deadline not armed: out of memory", + &app->name); + + return NULL; + } + + nxt_memzero(st, sizeof(nxt_router_start_timer_t)); + + st->app_name.start = nxt_pointer_to(st, sizeof(nxt_router_start_timer_t)); + st->app_name.length = app->name.length; + nxt_memcpy(st->app_name.start, app->name.start, app->name.length); + + st->timeout = timeout; + st->port = port; + st->dport = dport; + st->reply_port = port->id; + st->buf = b; + + nxt_port_inc_use(port); + nxt_port_inc_use(dport); + + /* The reference of the caller, which hands it to the RPC. */ + st->refs = 1; + + if (b == NULL) { + /* + * Nothing to wait for: a message with no payload and no descriptors + * (the worker-start leg, where the prototype is already up) leaves + * nothing of ours reachable from the send queue, so expiry can fail + * the RPC directly. + */ + st->send_done = 1; + + } else { + /* The reference of the pending buffer completion. */ + st->refs++; + + b->parent = st; + } + + return st; +} + + +/* + * Arm it. Separate from creation because the stream is only known once the + * RPC is registered, and because a write that fails must be able to drop the + * struct without ever having put a node in the engine's timer tree. + */ + +static void +nxt_router_start_timer_arm(nxt_task_t *task, nxt_router_start_timer_t *st, + uint32_t stream) +{ + nxt_event_engine_t *engine; + + st->stream = stream; + + engine = task->thread->engine; + + st->timer.bias = NXT_TIMER_DEFAULT_BIAS; + st->timer.work_queue = &engine->fast_work_queue; + st->timer.handler = nxt_router_start_timeout; + st->timer.task = &engine->task; + st->timer.log = st->timer.task->log; + + /* The reference of the engine, while the timer node is live. */ + st->refs++; + st->armed = 1; + + nxt_timer_add(engine, &st->timer, st->timeout); + + nxt_debug(task, "app \"%V\" stream #%uD start deadline %M ms", + &st->app_name, stream, st->timeout); +} + + +static void +nxt_router_start_timer_use(nxt_task_t *task, nxt_router_start_timer_t *st, + int32_t delta) +{ + st->refs += delta; + + nxt_assert(st->refs >= 0); + + if (st->refs > 0) { + return; + } + + nxt_port_use(task, st->port, -1); + nxt_port_use(task, st->dport, -1); + + nxt_free(st); +} + + +/* + * The start answered, or its write failed: drop the RPC's reference and, with + * it, the deadline. + * + * nxt_timer_disable() is not enough -- it clears the enabled bit but leaves + * the node in the engine's rbtree, which would dangle the moment this struct + * is freed. nxt_timer_delete() removes it, but its removal can itself be a + * queued change referencing the timer, so a non-zero return means the struct + * is still reachable from the engine: the engine's reference is then handed to + * a release handler instead of being dropped here. Re-arming at zero is the + * same trick nxt_router_free_app() uses for app_joint->idle_timer. + * + * When the expiry handler is the caller's own caller, ->armed is already + * clear: the node has fired, so it owns the engine's reference and drops it + * itself. + */ + +static void +nxt_router_start_timer_cancel(nxt_task_t *task, nxt_router_start_timer_t *st) +{ + nxt_event_engine_t *engine; + + st->retired = 1; + + if (st->armed) { + engine = task->thread->engine; + + st->armed = 0; + + if (nxt_timer_delete(engine, &st->timer)) { + st->timer.handler = nxt_router_start_timer_release; + nxt_timer_add(engine, &st->timer, 0); + + } else { + nxt_router_start_timer_use(task, st, -1); + } + } + + nxt_router_start_timer_use(task, st, -1); +} + + +static void +nxt_router_start_timer_release(nxt_task_t *task, void *obj, void *data) +{ + nxt_timer_t *timer; + + timer = obj; + + nxt_router_start_timer_use(task, + nxt_timer_data(timer, nxt_router_start_timer_t, timer), -1); +} + + +/* + * The deadline fired. What it may do depends on where the START_PROCESS it is + * giving up on has got to; see the comment on nxt_router_start_timer_s. + */ + +static void +nxt_router_start_timeout(nxt_task_t *task, void *obj, void *data) +{ + nxt_timer_t *timer; + nxt_port_msg_cancel_t cancelled; + nxt_router_start_timer_t *st; + + timer = obj; + st = nxt_timer_data(timer, nxt_router_start_timer_t, timer); + + /* + * "unless a reply lands first" is not hedging. The failure is driven + * from the payload's completion, which for a partly sent message waits + * for the rest of it to leave (see below), and a PROCESS_READY that + * arrives in that window retires the stream before the deadline can. + */ + + nxt_alert(task, "app \"%V\" process did not become ready in time; the " + "deadline expired, and the start request (stream #%uD) " + "will be failed unless a reply lands first. The process, " + "if it is still running, never called nxt_unit_init() -- " + "raise \"limits\": {\"start_timeout\"} if the application " + "simply needs longer to start", + &st->app_name, st->stream); + + /* + * The node has fired, so it is out of the engine's tree: clearing ->armed + * both records that and keeps the error handler reached below from trying + * to delete it again through nxt_router_start_timer_cancel(). The + * engine's reference is this frame's now, and is dropped at the end. + */ + + st->armed = 0; + st->expired = 1; + + /* + * Take the message back if there is still anything to take back. Both + * shapes of START_PROCESS go through this, for different reasons. + * + * The one that carries a payload (a prototype start) must be recalled + * before its RPC can be failed at all: failing it releases the pool the + * payload lives in and the descriptors it borrows. + * + * The header-only one (a worker start, sent to a prototype that is already + * up) carries no router memory and no descriptors, so nothing about it is + * a lifetime hazard. Recalling it is still worth doing: delivered after + * the stream is retired, it starts a worker whose PROCESS_READY answers + * nobody -- nxt_port_rpc_handler() drops an unknown stream -- leaving a + * process and a port the router is not tracking. That is the same orphan + * the deadline already leaves when a worker is merely slow, since the + * router never learns its pid and so cannot kill it either, and reaping in + * the prototype is what will collect both; but there is no reason to + * create one while the message has not left yet. + * + * A destination whose port has a shared-memory queue is the case this + * cannot reach: nxt_port_socket_write2() puts the message straight into + * the ring, where it is already delivered, and only a READ_QUEUE + * notification can be left in port->messages. Matching on the message + * type is what keeps this from recalling that notification and stranding a + * message that is genuinely on its way. + */ + + cancelled = nxt_port_socket_cancel(task, st->dport, + NXT_PORT_MSG_START_PROCESS, + st->stream, st->reply_port, st->buf); + + /* + * Recalled before the destination saw it: no worker was forked for this + * start, so there is no process to account for and the slot goes back the + * ordinary way. Accounting it would move a slot to unaccounted_processes + * that nothing can ever drain -- no PROCESS_READY and no REMOVE_PID are + * coming for a process that was never created -- which would turn the + * bound into a ratchet. See nxt_router_app_port_error(). + */ + + st->recalled = (cancelled == NXT_PORT_MSG_CANCELLED); + + if (st->send_done) { + /* + * Nothing of ours is in the port layer's hands -- either the payload's + * completion has already run, or there was never a payload -- so the + * RPC is failed here. There is nothing left to drive it otherwise: a + * header-only message has no buffers for the recall to complete. The + * stream may meanwhile have been retired by a reply that landed in the + * same turn of the event loop; nxt_port_rpc_error() drops an unknown + * stream, so the race costs a debug line. + */ + + nxt_port_rpc_error(task, st->port, st->stream); + + } else { + /* + * A payload still in flight: whichever of the three states it is in, + * the RPC is failed from nxt_router_start_buf_completion(). + * NXT_PORT_MSG_CANCELLED has just queued that completion, and the + * other two mean the message is still being sent, so the completion is + * still to come. Waiting is what keeps nxt_router_conf_error() from + * releasing a payload the send queue still points at. + * + * NXT_PORT_MSG_STARTED is the limitation of a protocol without + * cancellation: a destination that took one fragment and then stopped + * draining holds the failure off for as long as it stays stuck, and it + * is worth saying so, because from the outside that looks like a + * deadline that did not fire. The wedge this deadline exists for -- a + * destination that is draining fine and a worker that never announces + * itself -- is unaffected, because there the message goes out whole. + */ + + if (cancelled == NXT_PORT_MSG_STARTED) { + nxt_alert(task, "app \"%V\" stream #%uD start request is partly " + "sent; failing it has to wait for the rest of it " + "to leave, and a reply that lands meanwhile wins", + &st->app_name, st->stream); + } + } + + /* The engine's reference: the timer node is gone. */ + + nxt_router_start_timer_use(task, st, -1); +} + + +/* + * Which way it left: the port layer advances the payload as it sends it, so + * bytes still unconsumed at its completion mean the destination never saw the + * whole message. It was taken back by nxt_port_socket_cancel(), or dropped by + * nxt_port_msg_drop() because the destination died or the port layer could not + * requeue it -- and nothing was forked for a start that never arrived. + * + * ->recalled cannot answer this on its own: a message the deadline finds + * partly sent stays queued, and only its completion knows whether the rest of + * it ever left. See nxt_router_app_port_error(). + */ + +nxt_inline nxt_bool_t +nxt_router_start_buf_dropped(nxt_buf_t *b) +{ + return nxt_buf_mem_used_size(&b->mem) != 0; +} + + +/* + * The payload has left the port layer, one way or another: written in full, + * dropped by nxt_port_socket_cancel(), or released by nxt_port_error_handler() + * when the destination died. This is the only point at which nothing in + * dport->messages can reach the buffer or the borrowed descriptors any more, + * so it is where an expired start is failed. + */ + +static void +nxt_router_start_buf_completion(nxt_task_t *task, nxt_router_start_timer_t *st, + nxt_bool_t dropped) +{ + nxt_bool_t fail; + + st->send_done = 1; + st->buf = NULL; + st->dropped = dropped; + + fail = (st->expired && !st->retired); + + if (fail) { + /* + * Nothing may touch b after this: failing the RPC reaches + * nxt_router_conf_error(), which releases the pool the buffer was + * allocated from. + */ + + nxt_port_rpc_error(task, st->port, st->stream); + } + + nxt_router_start_timer_use(task, st, -1); +} + + +/* + * The config-apply payload is allocated from tmcf->mem_pool and freed with it, + * so this replaces nxt_buf_dummy_completion() and frees nothing. + */ + +static void +nxt_router_start_conf_buf_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_router_start_buf_completion(task, data, + nxt_router_start_buf_dropped(obj)); +} + + +/* + * The on-demand payload is allocated from the engine's pool and owns itself, + * so this replaces nxt_buf_completion(): it returns the buffer to that pool + * first, because failing the RPC below may not return at all cheaply. The + * buffer never has a parent, so none of nxt_buf_parent_completion()'s work + * applies -- which is also why the context travels in b->parent and this + * handler is not nxt_buf_completion(). + */ + +static void +nxt_router_start_app_buf_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_buf_t *b; + nxt_bool_t dropped; + nxt_router_start_timer_t *st; + + b = obj; + st = data; + + nxt_assert(b->next == NULL); + nxt_assert(b->parent == st); + + /* Read before the free: the answer is in the buffer's own pointers. */ + + dropped = nxt_router_start_buf_dropped(b); + + nxt_mp_free(b->data, b); + + nxt_router_start_buf_completion(task, st, dropped); +} + + + static void nxt_router_app_joint_use(nxt_task_t *task, nxt_app_joint_t *app_joint, int i) { @@ -1142,6 +1719,7 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) app_stat->active_requests = app->active_requests; app_stat->pending_processes = app->pending_processes; app_stat->processes = app->processes; + app_stat->unaccounted_processes = app->unaccounted_processes; app_stat->idle_processes = app->idle_processes; report->apps_count++; @@ -1283,6 +1861,25 @@ nxt_router_temp_conf(nxt_task_t *task) } +#if (NXT_TESTS) + +/* + * src/test/nxt_router_start_timeout_test.c drives the config-apply start + * against a real temporary configuration, because that is the one whose + * failure path releases the pool the START_PROCESS payload lives in. Building + * one by hand would not do: nxt_router_temp_conf() is also what initialises the + * socket queues nxt_router_conf_error() walks. + */ + +nxt_router_temp_conf_t * +nxt_router_test_temp_conf(nxt_task_t *task) +{ + return nxt_router_temp_conf(task); +} + +#endif + + nxt_inline nxt_bool_t nxt_router_app_need_start(nxt_app_t *app) { @@ -1604,6 +2201,12 @@ static nxt_conf_map_t nxt_router_app_limits_conf[] = { NXT_CONF_MAP_MSEC, offsetof(nxt_router_app_conf_t, timeout), }, + + { + nxt_string("start_timeout"), + NXT_CONF_MAP_MSEC, + offsetof(nxt_router_app_conf_t, start_timeout), + }, }; @@ -2027,6 +2630,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, apcf.max_processes = 1; apcf.spare_processes = 0; apcf.timeout = 0; + apcf.start_timeout = NXT_APP_START_TIMEOUT; apcf.idle_timeout = 15000; apcf.limits_value = NULL; apcf.processes_value = NULL; @@ -2136,6 +2740,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, app->max_pending_processes = apcf.spare_processes ? apcf.spare_processes : 1; app->timeout = apcf.timeout; + app->start_timeout = apcf.start_timeout; app->idle_timeout = apcf.idle_timeout; app->targets = targets; @@ -3361,17 +3966,20 @@ static void nxt_router_app_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_app_t *app) { - size_t size; - uint32_t stream; - nxt_fd_t port_fd, queue_fd; - nxt_int_t ret; - nxt_buf_t *b; - nxt_port_t *router_port, *dport; - nxt_runtime_t *rt; - nxt_app_rpc_t *rpc; + size_t size; + uint32_t stream; + nxt_fd_t port_fd, queue_fd; + nxt_int_t ret; + nxt_buf_t *b; + nxt_port_t *router_port, *dport; + nxt_runtime_t *rt; + nxt_app_rpc_t *rpc; + nxt_router_start_timer_t *st; rt = task->thread->runtime; + st = NULL; + dport = app->proto_port; if (dport == NULL) { @@ -3419,6 +4027,17 @@ nxt_router_app_rpc_create(nxt_task_t *task, stream = nxt_port_rpc_ex_stream(rpc); + /* + * Before the write: the deadline resolves through the payload's completion + * handler, which nxt_port_socket_write2() may run before it returns. + */ + + st = nxt_router_start_timer_create(task, app, router_port, dport, b); + + if (st != NULL && b != NULL) { + b->completion_handler = nxt_router_start_conf_buf_completion; + } + ret = nxt_port_socket_write2(task, dport, NXT_PORT_MSG_START_PROCESS, port_fd, queue_fd, stream, router_port->id, b); if (nxt_slow_path(ret != NXT_OK)) { @@ -3432,14 +4051,45 @@ nxt_router_app_rpc_create(nxt_task_t *task, app->pending_processes++; } + rpc->start_timer = st; + + if (st != NULL) { + nxt_router_start_timer_arm(task, st, stream); + } + return; fail: + if (st != NULL) { + /* + * Never armed, and the payload dies with tmcf->mem_pool below without + * its completion handler ever running, so both references go here. + */ + + if (b != NULL) { + nxt_router_start_timer_use(task, st, -1); + } + + nxt_router_start_timer_use(task, st, -1); + } + nxt_router_conf_error(task, tmcf); } +#if (NXT_TESTS) + +void +nxt_router_test_app_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, + nxt_app_t *app) +{ + nxt_router_app_rpc_create(task, tmcf, app); +} + +#endif + + static void nxt_router_app_prefork_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) @@ -3452,6 +4102,12 @@ nxt_router_app_prefork_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, rpc = data; app = rpc->app; + if (rpc->start_timer != NULL) { + nxt_router_start_timer_cancel(task, rpc->start_timer); + + rpc->start_timer = NULL; + } + port = msg->u.new_port; nxt_assert(port != NULL); @@ -3514,6 +4170,12 @@ nxt_router_app_prefork_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, app = rpc->app; tmcf = rpc->temp_conf; + if (rpc->start_timer != NULL) { + nxt_router_start_timer_cancel(task, rpc->start_timer); + + rpc->start_timer = NULL; + } + if (rpc->proto) { nxt_log(task, NXT_LOG_WARN, "failed to start prototype \"%V\"", &app->name); @@ -5066,7 +5728,7 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, { uint32_t n; nxt_app_t *app; - nxt_bool_t start_process, restarted; + nxt_bool_t start_process, restarted, superseded; nxt_port_t *port; nxt_app_joint_t *app_joint; nxt_app_joint_rpc_t *app_joint_rpc; @@ -5077,6 +5739,12 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, app_joint = app_joint_rpc->app_joint; port = msg->u.new_port; + if (app_joint_rpc->start_timer != NULL) { + nxt_router_start_timer_cancel(task, app_joint_rpc->start_timer); + + app_joint_rpc->start_timer = NULL; + } + nxt_assert(app_joint != NULL); nxt_assert(port != NULL); nxt_assert(port->id == 0); @@ -5098,16 +5766,49 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, restarted = (app->generation != app_joint_rpc->generation); if (app_joint_rpc->proto) { - nxt_assert(app->proto_port == NULL); nxt_assert(port->type == NXT_PROCESS_PROTOTYPE); - n = app->proto_port_requests; - app->proto_port_requests = 0; + if (app_joint_rpc->expired) { + /* + * Not a start attempt any more: the reaper armed by + * nxt_router_app_start_expired() for the prototype the deadline + * gave up on, which has announced itself after all. The cohort + * it owned was released when the attempt was failed, so there is + * nothing to replay here, and the slot it holds is an + * unaccounted one. Guarded rather than asserted for the reason + * given below, where a worker takes the same step. + */ + + nxt_assert(app->unaccounted_processes != 0); + + if (nxt_fast_path(app->unaccounted_processes != 0)) { + app->unaccounted_processes--; + } + + n = 0; + + } else { + n = app->proto_port_requests; + app->proto_port_requests = 0; + } + + /* + * A prototype that only just missed its deadline is adopted, exactly + * as a late worker is below -- unless the application has acquired a + * prototype meanwhile. It can: failing the attempt clears + * proto_port_requests, so the next request starts a second prototype + * and two of them can be up at once, of which only one may ever be + * app->proto_port. The other is QUIT here, which is also how the + * router disposes of one that belongs to a superseded generation. + */ + + superseded = restarted || app->proto_port != NULL; - if (nxt_slow_path(restarted)) { + if (nxt_slow_path(superseded)) { nxt_thread_mutex_unlock(&app->mutex); - nxt_debug(task, "proto port ready for restarted app, send QUIT"); + nxt_debug(task, "proto port ready for an app that has no use for " + "it, send QUIT"); nxt_port_socket_write(task, port, NXT_PORT_MSG_QUIT, -1, 0, 0, NULL); @@ -5135,9 +5836,41 @@ nxt_router_app_port_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, } nxt_assert(port->type == NXT_PROCESS_APP); - nxt_assert(app->pending_processes != 0); - app->pending_processes--; + if (app_joint_rpc->expired) { + /* + * The worker beat the reaper: "limits": {"start_timeout"} gave up on + * its start and moved the slot to unaccounted_processes, and the + * process the router had no pid for has now announced itself. Move + * the slot back and adopt the worker exactly as an in-time start is + * adopted below -- the sum nxt_router_app_can_start() is taken over + * does not change, and this is the only drain that recovers a slot + * without the process having to die. The request that waited for it + * was answered when the deadline fired; the worker serves the next + * one. + */ + + /* + * Guarded rather than asserted, for the reason given in + * nxt_router_app_start_failed(): nxt_assert() is nothing in a release + * build, and of the two ways to be wrong here, decrementing a counter + * that is already zero is much the worse. It wraps to UINT32_MAX and + * pins nxt_router_app_can_start() false for the application's + * lifetime, where failing to decrement merely leaks the slot. + * nxt_router_app_unaccounted_release() takes the same care. + */ + + nxt_assert(app->unaccounted_processes != 0); + + if (nxt_fast_path(app->unaccounted_processes != 0)) { + app->unaccounted_processes--; + } + + } else { + nxt_assert(app->pending_processes != 0); + + app->pending_processes--; + } if (nxt_slow_path(restarted)) { nxt_debug(task, "new port ready for restarted app, send QUIT"); @@ -5184,7 +5917,7 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { uint32_t n; - nxt_bool_t restarted; + nxt_bool_t reap, restarted, expired, unsent; nxt_app_t *app; nxt_app_joint_t *app_joint; nxt_app_joint_rpc_t *app_joint_rpc; @@ -5194,11 +5927,54 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, app_joint_rpc = data; app_joint = app_joint_rpc->app_joint; + expired = 0; + unsent = 0; + + if (app_joint_rpc->start_timer != NULL) { + /* Read before the cancel, which is what drops the deadline. */ + expired = app_joint_rpc->start_timer->expired; + + /* The two ways the destination never saw the whole message. */ + unsent = app_joint_rpc->start_timer->recalled + || app_joint_rpc->start_timer->dropped; + + nxt_router_start_timer_cancel(task, app_joint_rpc->start_timer); + + app_joint_rpc->start_timer = NULL; + } + nxt_assert(app_joint != NULL); app = app_joint->app; - nxt_router_app_joint_use(task, app_joint, -1); + /* + * A start the deadline gave up on keeps its stream watched, and the + * reaper that watches it inherits this attempt's reference rather than + * taking one of its own -- so that the joint cannot be released between + * the two. + * + * Both shapes of start are reaped, because both leave a process behind. + * A worker start leaves the worker the prototype forked for it. A + * prototype start leaves the prototype itself: main forks it before it + * can answer, and one that never reaches PROCESS_READY forks no worker, + * so exactly one process comes of the attempt whatever the size of the + * cohort parked on it. The cohort stands for requests, not for + * processes; only the reaped slot stands for a process. + * + * A start the destination never saw whole is the exception -- recalled + * out of the send queue, or dropped by the port layer with part of it + * still unsent. Nothing was forked for either, so there is nothing to + * reap and the slot goes back the ordinary way. Accounting one of those + * would move a slot to unaccounted_processes that nothing can drain: no + * PROCESS_READY and no REMOVE_PID are coming for a process that was + * never created, which would make the bound a ratchet. + */ + + reap = (expired && !unsent && app != NULL); + + if (!reap) { + nxt_router_app_joint_use(task, app_joint, -1); + } if (nxt_slow_path(app == NULL)) { nxt_debug(task, "start error for released app"); @@ -5206,6 +5982,19 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, return; } + if (app_joint_rpc->expired) { + /* + * Not a start attempt: the reaper armed by + * nxt_router_app_start_expired() for a process the router had no + * pid for. Being here is the whole point -- the process is gone, + * and its unaccounted_processes slot goes with it. + */ + + nxt_router_app_unaccounted_release(task, app); + + return; + } + /* * Two ways in are not failures at all, and both are excluded rather * than tolerated. @@ -5217,10 +6006,11 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, * set it when it forked (nxt_application.c) and nothing clears it for * a process that never became ready -- so the router retypes it into * an RPC error and it lands here, reporting a start the operator - * themselves cancelled. Nothing about the RPC peer is involved: the - * registration this handler belongs to never sets one. Generation is - * how nxt_router_app_port_ready() tells the same two apart before it - * quietly QUITs a port that arrives for a superseded one. + * themselves cancelled. The RPC peer is not what brings it here: that + * is the process the START_PROCESS was sent to, and this arrives for + * the worker. Generation is how nxt_router_app_port_ready() tells the + * same two apart before it quietly QUITs a port that arrives for a + * superseded one. * * And a shutdown: nxt_port_close() runs nxt_port_rpc_close() over the * router port, which turns every registration still on it into an @@ -5277,6 +6067,20 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, if (nxt_slow_path(restarted || task->thread->engine->shutdown)) { nxt_debug(task, "app '%V' start attempt cancelled", &app->name); + } else if (expired) { + /* + * A third way in that is not what the line above describes: the + * "limits": {"start_timeout"} deadline failed this RPC itself. The + * process it was waiting for very probably does exist -- the router + * never learns its pid, so it cannot say -- it simply never called + * nxt_unit_init(). "produced no process" would be the wrong thing + * to send an operator looking through the logs, and the deadline has + * already said the right one, naming the application and the knob. + */ + + nxt_debug(task, "app '%V' start attempt bounded by \"start_timeout\"", + &app->name); + } else { nxt_log(task, NXT_LOG_ERR, "app '%V' start attempt produced no process", &app->name); @@ -5291,11 +6095,14 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, * and park without sending anything -- so no further port_ready or * port_error can ever fire to clear it. Only replacing the * nxt_app_t recovers, and a reload does that only when the - * application's own config text changes. Clear it here, and - * release the slots of the whole parked cohort: the initiator's, - * plus one for each caller that joined the wait. This mirrors - * port_ready(), which takes the same count and replays that many - * starts on success. + * application's own config text changes. Clear it here, and take + * the whole parked cohort: the initiator's slot, plus one for each + * caller that joined the wait. This mirrors port_ready(), which + * takes the same count and replays that many starts on success. + * + * Where those slots go is decided below. All of them go back when + * the attempt forked nothing; when it did, one of them stays behind + * with the prototype, which is the process it forked. */ nxt_thread_mutex_lock(&app->mutex); @@ -5315,7 +6122,26 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, n = 1; } - nxt_router_app_start_failed(task, app, n); + if (reap) { + /* + * One process was forked for this attempt, and the reaper accounts + * for it. The rest of a prototype start's cohort goes back the + * ordinary way: those slots were taken by requests that joined the + * wait, and no process was forked for any of them. + */ + + if (n > 1) { + nxt_router_app_start_failed(task, app, n - 1, 0); + } + + nxt_router_app_start_expired(task, app, msg, app_joint, + app_joint_rpc->generation, + app_joint_rpc->proto); + + return; + } + + nxt_router_app_start_failed(task, app, n, 0); } @@ -5327,11 +6153,19 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, * * "count" is 1 for an ordinary attempt, which owns only its initiator's slot. * A failed prototype attempt owns the whole parked cohort: see - * nxt_router_app_port_error(). + * nxt_router_app_port_error(). Zero is the caller that owns no slot at all + * and only wants the requests answered: nxt_router_app_port_get(), for a + * request that arrives when the application can neither serve it nor start + * anything to serve it. + * + * "unaccounted" moves the slots to app->unaccounted_processes instead of + * giving them back, for the one caller whose attempt very probably did leave + * an OS process behind: nxt_router_app_start_expired(). */ static void -nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, uint32_t count) +nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, uint32_t count, + nxt_bool_t unaccounted) { nxt_queue_link_t *link; nxt_http_request_t *r; @@ -5355,6 +6189,10 @@ nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, uint32_t count) app->pending_processes -= count; + if (unaccounted) { + app->unaccounted_processes += count; + } + if (app->processes == 0 && !nxt_queue_is_empty(&app->ack_waiting_req)) { link = nxt_queue_first(&app->ack_waiting_req); @@ -5387,6 +6225,138 @@ nxt_router_app_start_failed(nxt_task_t *task, nxt_app_t *app, uint32_t count) } +/* + * A start that "limits": {"start_timeout"} gave up on. + * + * The attempt is over -- its requests are answered here, through + * nxt_router_app_start_failed() -- but unlike every other way an attempt can + * end, this one very probably leaves a process running. Something was forked + * (only a forked process can fail to announce itself), and the router has no + * pid for it, because a pid is exactly what PROCESS_READY carries. Giving + * the pending_processes slot back would therefore mean the next request forks + * another one, with nothing counting either: "processes": {"max"} would bound + * only the processes that work. The slot moves to unaccounted_processes + * instead, where nxt_router_app_can_start() still sees it. + * + * "proto" says what was forked, and only nxt_router_app_port_ready() cares: + * a worker start leaves the worker its prototype forked, a prototype start + * leaves the prototype main forked for it. A stuck prototype forks no worker + * of its own, so either way the attempt leaves exactly one process, and one + * slot accounts for it. That a stuck prototype spends worker slots is + * deliberate: while it is stuck the application has no way to run a worker + * anyway, and the alternative -- a counter of its own -- would let prototypes + * and workers each grow to "max". + * + * That is a bound, not a leak, because the stream stays watched. The reply + * this start gave up on can still arrive, and both shapes of it retire the + * slot: + * + * - the process announces itself late, missing only the deadline; + * nxt_router_app_port_ready() retires the slot and adopts it -- a worker + * into ->processes, a prototype into ->proto_port unless the application + * has one by then -- so a merely slow application costs one 503 and + * nothing else; + * + * - the process dies and the router is told with the start's own stream + * still attached: for a worker by the prototype that forked it and + * reaps it, for a prototype by main, which clears process->stream only + * for a process that reached READY (see nxt_port_remove_notify_others(), + * and nxt_router_remove_pid_handler(), which retypes that REMOVE_PID + * into an RPC error) -- and nxt_router_app_port_error() gives the slot + * back. + * + * Neither is guaranteed: a process wedged forever is exactly the case this + * exists for, and it holds its slot for as long as it lives. That is the + * intended trade. A configuration reload that replaces the nxt_app_t is the + * other way back. + */ + +static void +nxt_router_app_start_expired(nxt_task_t *task, nxt_app_t *app, + nxt_port_recv_msg_t *msg, nxt_app_joint_t *app_joint, uint32_t generation, + nxt_bool_t proto) +{ + uint32_t stream; + nxt_app_joint_rpc_t *reaper; + + stream = msg->port_msg.stream; + + nxt_router_app_start_failed(task, app, 1, 1); + + /* + * Only from inside this stream's own handler is the key free to take + * again; see nxt_port_rpc_register_handler_at(). + */ + + reaper = nxt_port_rpc_register_handler_at(task, msg->port, + nxt_router_app_port_ready, + nxt_router_app_port_error, + stream, + sizeof(nxt_app_joint_rpc_t)); + + if (nxt_slow_path(reaper == NULL)) { + /* + * The bound holds -- the slot is already counted -- but nothing will + * hand it back, so say so: from here only a reload recovers it. + */ + + nxt_alert(task, "app \"%V\" cannot watch the process its expired " + "start left behind; that \"processes\" slot is held " + "until the application is reconfigured", &app->name); + + nxt_router_app_joint_use(task, app_joint, -1); + + return; + } + + /* The reaper inherits the failed attempt's app_joint reference. */ + + reaper->app_joint = app_joint; + reaper->generation = generation; + reaper->expired = 1; + + /* + * Which kind of process the slot stands for: nxt_router_app_port_ready() + * branches on ->proto before anything else, and a late prototype must + * not be adopted as though it were a worker. + */ + + reaper->proto = proto; + + nxt_debug(task, "app '%V' stream #%uD kept to account for the process an " + "expired start left behind", &app->name, stream); +} + + +/* + * The process an expired start left behind is accounted for at last: it died + * and the router was told. Only nxt_router_app_start_expired() puts a slot + * here, and only this and nxt_router_app_port_ready() take one back. + */ + +static void +nxt_router_app_unaccounted_release(nxt_task_t *task, nxt_app_t *app) +{ + nxt_thread_mutex_lock(&app->mutex); + + /* + * One reaper owns exactly one slot, so this cannot underflow; the test + * is here because a wrapped counter would pin nxt_router_app_can_start() + * false for the application's lifetime, which is worse than leaking the + * slot it is meant to release. + */ + + if (nxt_fast_path(app->unaccounted_processes != 0)) { + app->unaccounted_processes--; + } + + nxt_thread_mutex_unlock(&app->mutex); + + nxt_debug(task, "app '%V' unaccounted process is gone, %d left", + &app->name, app->unaccounted_processes); +} + + nxt_inline nxt_port_t * nxt_router_app_get_port_for_quit(nxt_task_t *task, nxt_app_t *app) { @@ -5881,7 +6851,7 @@ static void nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, nxt_request_rpc_data_t *req_rpc_data) { - nxt_bool_t start_process; + nxt_bool_t start_process, unanswerable; nxt_port_t *port; nxt_http_request_t *r; @@ -5899,6 +6869,21 @@ nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, start_process = 1; } + /* + * Nothing is running, nothing is starting, and nothing may be started: + * every "processes" slot is held by a process an expired start left + * behind (nxt_router_app_start_expired()). Only a start can take a + * request back out of ack_waiting_req, so parking this one would leave + * it there until it timed out. Answer it below instead, at once. + * + * Reachable only through unaccounted_processes: with no process and + * none pending, nxt_router_app_can_start() is otherwise false only for + * "processes": {"max": 0}, which the configuration rejects. + */ + + unanswerable = (start_process == 0 && app->processes == 0 + && app->pending_processes == 0); + r = req_rpc_data->request; /* @@ -5920,6 +6905,14 @@ nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, if (start_process) { nxt_router_start_app_process(task, app); + + } else if (nxt_slow_path(unanswerable)) { + nxt_debug(task, "app '%V' has no process and may not start one", + &app->name); + + /* No slot of its own to give back: only the requests are wanted. */ + + nxt_router_app_start_failed(task, app, 0, 0); } } diff --git a/src/nxt_router.h b/src/nxt_router.h index ab3f6e54d..9f1844edd 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -102,6 +102,28 @@ typedef struct { } nxt_joint_job_t; +/* + * Default "limits": {"start_timeout"}, in milliseconds: how long a worker may + * take to reach nxt_unit_init() before its start request is failed. Zero + * disables the bound, and that is the default deliberately. + * + * The window this measures is the whole of the application's own startup, not + * just Unit's part of it: every module runs user code before nxt_unit_init() + * -- nxt_python_start() imports the application module + * (src/python/nxt_python.c:303, via nxt_python_set_target()) before + * nxt_unit_init() (:365), nxt_java_start() deploys the webapp + * (src/nxt_java.c:417) before :441, the Node module loads the script before + * src/nodejs/unit-http/unit.cpp:279, and Go and "external" run main() first. + * A default bound would + * therefore fail a Django or ML import, or a JVM webapp deployment, that + * works today -- and a start failed while the worker is merely slow leaves + * that worker alive and orphaned, because the router cannot kill a process it + * has no pid for. So the bound is opt-in, and choosing a value is the + * operator's judgement about their own application's startup. + */ +#define NXT_APP_START_TIMEOUT 0 + + typedef struct { uint32_t use_count; nxt_app_t *app; @@ -129,6 +151,21 @@ struct nxt_app_s { uint32_t processes; uint32_t idle_processes; + /* + * Application processes the router asked for, that were forked, and that + * it has neither a port nor a pid for: the ones a "limits": + * {"start_timeout"} deadline gave up on. Such a worker is not in + * ->processes, because only PROCESS_READY puts one there, and it is no + * longer in ->pending_processes, because its start attempt is over -- + * so without a third counter "processes": {"max"} would bound nothing, + * and every expired start would fork another OS child. + * + * A slot leaves again when the process it stands for is finally + * accounted for: it announces itself late, or it dies and the router is + * told. See nxt_router_app_start_expired() in src/nxt_router.c. + */ + uint32_t unaccounted_processes; + uint32_t max_processes; uint32_t spare_processes; uint32_t max_pending_processes; @@ -139,6 +176,14 @@ struct nxt_app_s { nxt_msec_t timeout; nxt_msec_t idle_timeout; + /* + * Bound on a START_PROCESS RPC: a worker that is forked but never calls + * nxt_unit_init() sends no PROCESS_READY, so without this nothing ever + * retires the RPC. Zero disables the bound. See + * nxt_router_start_timer_t in nxt_router.c. + */ + nxt_msec_t start_timeout; + nxt_str_t *targets; nxt_app_type_t type:8; @@ -170,7 +215,8 @@ struct nxt_app_s { nxt_inline nxt_bool_t nxt_router_app_can_start(nxt_app_t *app) { - return app->processes + app->pending_processes < app->max_processes + return app->processes + app->pending_processes + + app->unaccounted_processes < app->max_processes && app->pending_processes < app->max_pending_processes; } @@ -288,6 +334,16 @@ void nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, void nxt_router_access_log_reopen_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +#if (NXT_TESTS) +/* + * The config-apply start and the temporary configuration its failure path + * releases, for the start-deadline test. + */ +nxt_router_temp_conf_t *nxt_router_test_temp_conf(nxt_task_t *task); +void nxt_router_test_app_rpc_create(nxt_task_t *task, + nxt_router_temp_conf_t *tmcf, nxt_app_t *app); +#endif + extern nxt_router_t *nxt_router; diff --git a/src/nxt_status.c b/src/nxt_status.c index e3ecb9201..0eb0c696a 100644 --- a/src/nxt_status.c +++ b/src/nxt_status.c @@ -42,6 +42,7 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) static const nxt_str_t procs_str = nxt_string("processes"); static const nxt_str_t run_str = nxt_string("running"); static const nxt_str_t start_str = nxt_string("starting"); + static const nxt_str_t unacc_str = nxt_string("unaccounted"); /* * modules, connections, requests, applications -- plus "telemetry" when @@ -211,7 +212,7 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) return NULL; } - obj = nxt_conf_create_object(mp, 3); + obj = nxt_conf_create_object(mp, 4); if (nxt_slow_path(obj == NULL)) { return NULL; } @@ -220,7 +221,9 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) nxt_conf_set_member_integer(obj, &run_str, app->processes, 0); nxt_conf_set_member_integer(obj, &start_str, app->pending_processes, 1); - nxt_conf_set_member_integer(obj, &idle_str, app->idle_processes, 2); + nxt_conf_set_member_integer(obj, &unacc_str, + app->unaccounted_processes, 2); + nxt_conf_set_member_integer(obj, &idle_str, app->idle_processes, 3); obj = nxt_conf_create_object(mp, 1); if (nxt_slow_path(obj == NULL)) { diff --git a/src/nxt_status.h b/src/nxt_status.h index 8c370cba1..f8eb1c8ed 100644 --- a/src/nxt_status.h +++ b/src/nxt_status.h @@ -12,6 +12,7 @@ typedef struct { uint32_t active_requests; uint32_t pending_processes; uint32_t processes; + uint32_t unaccounted_processes; uint32_t idle_processes; } nxt_status_app_t; diff --git a/src/test/nxt_router_start_timeout_test.c b/src/test/nxt_router_start_timeout_test.c new file mode 100644 index 000000000..289792d3b --- /dev/null +++ b/src/test/nxt_router_start_timeout_test.c @@ -0,0 +1,1023 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the silent-worker start wedge: an application process + * that is forked successfully but never calls nxt_unit_init() sends no + * PROCESS_READY, so the START_PROCESS RPC the router armed for it was never + * retired. Nothing else could retire it either -- the on-demand start + * registers that RPC with no peer (nxt_router_start_app_process_handler()), + * and REMOVE_PID only arrives if the process dies, which a `/bin/sleep` does + * not. The armed handler owned an app->pending_processes slot and, on the + * config-apply path, was the sole continuation of nxt_router_conf_apply(), so + * the configuration PUT never returned and the controller queued every later + * request behind it -- GET /status included. + * + * The fix is a deadline armed with the START_PROCESS write and cancelled by + * whichever handler answers first. What this test pins down is that the + * deadline resolves the RPC *through the existing failure path* rather than + * inventing accounting of its own: on expiry the application must get its + * pending_processes slot back and its proto_port_requests cohort cleared, + * exactly as nxt_router_app_port_error() does for a start that fails for any + * other reason, and the application must be able to start a process again + * afterwards. + * + * The other half is the race the deadline creates: a reply that lands after + * the timer has been queued must not free the timer struct out from under the + * expiry handler, and a cancelled timer must not leave a node in the engine's + * rbtree. Both are driven here against a real nxt_timers_t, because a + * memzero'd engine would let a broken cancel look correct. + * + * The arrangement is the router's own: no prototype port yet, so the handler + * builds a START_PROCESS payload and arms an RPC with ->proto set; the message + * only queues (main's port has no queue and write_ready unset), so the RPC + * stays armed and nothing ever answers it. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* File scope: the handler hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_start_timeout_test_app; + + +/* + * Commit the pending timer changes and move what is due onto the work queue, + * the way nxt_event_engine_start() does, without running any of it: + * nxt_timer_find() folds the change list into the rbtree and + * nxt_timer_expire() queues the handlers that are due. Split out from the + * tick below so a leg can queue work *behind* an already-queued timer handler, + * which is the ordering nxt_port_write_handler() creates for real. + */ + +static void +nxt_router_start_timeout_test_expire(nxt_event_engine_t *engine, + nxt_msec_t advance) +{ + (void) nxt_timer_find(engine); + + nxt_timer_expire(engine, engine->timers.now + advance); +} + + +/* Drain the work queue, returning the number of handlers that ran. */ + +static nxt_uint_t +nxt_router_start_timeout_test_drain(nxt_task_t *task, + nxt_event_engine_t *engine) +{ + nxt_uint_t ran; + nxt_task_t *wq_task; + nxt_work_handler_t handler; + void *obj, *data; + + ran = 0; + + /* nxt_work_queue_pop() dereferences the head unconditionally. */ + + while (engine->fast_work_queue.head != NULL) { + handler = nxt_work_queue_pop(&engine->fast_work_queue, &wq_task, &obj, + &data); + + if (handler == NULL) { + continue; + } + + ran++; + + handler(wq_task != NULL ? wq_task : task, obj, data); + } + + return ran; +} + + +static nxt_uint_t +nxt_router_start_timeout_test_tick(nxt_task_t *task, nxt_event_engine_t *engine, + nxt_msec_t advance) +{ + nxt_router_start_timeout_test_expire(engine, advance); + + return nxt_router_start_timeout_test_drain(task, engine); +} + + +/* The message a start left in a port's send queue, if it is still there. */ + +static nxt_port_send_msg_t * +nxt_router_start_timeout_test_msg(nxt_port_t *port) +{ + nxt_queue_link_t *lnk; + + lnk = nxt_queue_first(&port->messages); + + if (lnk == nxt_queue_tail(&port->messages)) { + return NULL; + } + + return nxt_queue_link_data(lnk, nxt_port_send_msg_t, link); +} + + +nxt_int_t +nxt_router_start_timeout_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_uint_t ran; + uint32_t stream, joint_use; + nxt_bool_t app_mutex, timers; + nxt_task_t *task; + nxt_port_t *router_port, *main_port, *shared_port; + nxt_router_t test_router; + nxt_runtime_t *rt, *saved_rt; + nxt_atomic_t main_port_use; + nxt_app_joint_t *joint; + nxt_event_engine_t engine, *saved_engine; + nxt_port_send_msg_t *sent; + nxt_router_temp_conf_t *tmcf; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test started"); + + ret = NXT_ERROR; + router_port = NULL; + main_port = NULL; + shared_port = NULL; + joint = NULL; + app_mutex = 0; + timers = 0; + app = &nxt_router_start_timeout_test_app; + + task = thr->task; + task->thread = thr; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + joint = nxt_mp_zalloc(mp, sizeof(nxt_app_joint_t)); + if (nxt_slow_path(rt == NULL || joint == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + /* + * The deadline runs on the engine's own task, as it does in the router, + * so that task must be usable: a memzero'd one has a NULL log and the + * first nxt_debug() from nxt_timer_add() dereferences it. + */ + + engine.task.thread = thr; + engine.task.log = thr->log; + + /* + * A real timer machinery, not the memzero'd one the other router tests + * get away with: nxt_timer_add() writes into timers->changes, and a + * cancel that leaves a node in this rbtree is precisely the defect the + * teardown check below looks for. + */ + + if (nxt_slow_path(nxt_timers_init(&engine.timers, 64) != NXT_OK)) { + goto done; + } + + timers = 1; + + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port the handler registers its RPC on. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * A prototype start addresses the main process. No queue and write_ready + * unset, so nxt_port_socket_write2() takes the nxt_port_msg_chk_insert() + * path: the write succeeds, the message is queued and the RPC stays armed + * -- a start that nobody will ever answer, which is the case under test. + */ + + main_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + rt->port_by_type[NXT_PROCESS_MAIN] = main_port; + + shared_port = nxt_port_new(task, 2, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(shared_port == NULL)) { + goto done; + } + + shared_port->pair[0] = -1; + shared_port->pair[1] = -1; + shared_port->socket.fd = -1; + shared_port->queue_fd = -1; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "start-timeout-test"); + + app->joint = joint; + joint->app = app; + joint->use_count = 1; + + app->shared_port = shared_port; + app->max_processes = 8; + app->max_pending_processes = 4; + app->start_timeout = 1000; + + app->proto_port = NULL; + + /* One process alive, so the failure path has no requests to fail. */ + app->processes = 1; + + /* What a real initiator leaves behind: the slot and the work's use. */ + app->pending_processes = 1; + app->use_count = 2; + + /* Baseline for the reference-balance check after the deadline fires. */ + main_port_use = main_port->use_count; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 1 || joint->use_count != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: start left " + "proto_port_requests %uD joint %uD (expected 1 and 2)", + app->proto_port_requests, joint->use_count); + goto done; + } + + /* + * Well before the deadline nothing may fire: a deadline that expires + * early would fail starts that are merely slow. + */ + + ran = nxt_router_start_timeout_test_tick(task, &engine, 100); + + if (ran != 0 || app->pending_processes != 1 + || app->proto_port_requests != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: deadline fired early, " + "%ui handlers, pending %uD proto_port_requests %uD " + "(expected 0, 1 and 1)", + ran, app->pending_processes, app->proto_port_requests); + goto done; + } + + /* + * Past the deadline the RPC must resolve through the router's own error + * handler: the cohort cleared and every slot it owned given back. Before + * the fix nothing was armed at all, so this tick ran no handler and left + * both counters exactly as they were. + */ + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded the start, " + "the RPC is armed forever"); + goto done; + } + + if (app->proto_port_requests != 0 || app->pending_processes != 0 + || joint->use_count != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: expired start left " + "proto_port_requests %uD pending %uD joint %uD " + "(expected 0, 0 and 1)", + app->proto_port_requests, app->pending_processes, + joint->use_count); + goto done; + } + + /* + * And the start it gave up on is out of the send queue, with the reference + * nxt_port_msg_chk_insert() took for it discharged. This is the whole + * point of resolving through nxt_port_socket_cancel(): failing the RPC + * releases the payload and the application's shared-port descriptors, and + * a message left behind here would be dereferenced -- and its descriptors + * sent -- when the destination finally drained. + */ + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: an expired start left its " + "START_PROCESS in the destination's send queue"); + goto done; + } + + if (main_port->use_count != main_port_use) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: destination port use count " + "%uD after an expired start, expected %uD", + (uint32_t) main_port->use_count, + (uint32_t) main_port_use); + goto done; + } + + /* + * And the application is not wedged by its own deadline: with the cohort + * clear a later call must send a fresh prototype request -- arming an RPC + * and taking a joint reference -- rather than parking in the wait branch. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 1 || joint->use_count != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: app wedged after an expired " + "start, proto_port_requests %uD joint %uD " + "(expected 1 and 2)", + app->proto_port_requests, joint->use_count); + goto done; + } + + /* + * The other order: an answer arrives first. Closing the port's + * registrations drives nxt_router_app_port_error(), which must cancel the + * armed deadline. A cancel that only nxt_timer_disable()d it would leave + * a node in the rbtree pointing at freed memory; the tick below walks + * that tree, and the teardown check requires it empty. + */ + + nxt_port_rpc_close(task, router_port); + + if (app->proto_port_requests != 0 || app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: answered start left " + "proto_port_requests %uD pending %uD (expected 0 and 0)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * Two ticks: the first lets a deferred release run, the second must find + * nothing left. A cancelled deadline that still fired would report the + * application as failing a start it never made. + */ + + (void) nxt_router_start_timeout_test_tick(task, &engine, 2000); + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: %ui handlers still pending " + "after a cancelled deadline (expected 0)", ran); + goto done; + } + + if (!nxt_rbtree_is_empty(&engine.timers.tree)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a cancelled deadline left a " + "node in the engine's timer tree"); + goto done; + } + + /* + * The write/timer ordering inside one poll cycle. + * + * nxt_port_write_handler() removes a fully sent message from + * port->messages and only then queues its buffer completion, so the + * completion can land behind a timer handler that the same poll cycle + * queued ahead of it. The deadline then finds nothing in the send queue + * while the payload is still referenced by work that has not run: + * "not found" is not "safe to fail", and failing there would release the + * payload under the pending completion. + * + * Built here by expiring the timer first -- which only queues its handler + * -- and taking the message out afterwards, so its completion is queued + * behind that handler exactly as a real write would leave it. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: no message queued for the " + "same-poll ordering leg"); + goto done; + } + + stream = sent->port_msg.stream; + + nxt_router_start_timeout_test_expire(&engine, 2000); + + if (nxt_port_socket_cancel(task, main_port, NXT_PORT_MSG_START_PROCESS, + stream, router_port->id, NULL) + != NXT_PORT_MSG_CANCELLED) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: could not take the queued " + "start back for the same-poll ordering leg"); + goto done; + } + + if (app->proto_port_requests != 1 || app->pending_processes != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: taking the message back " + "failed the start by itself, proto_port_requests %uD " + "pending %uD (expected 1 and 1)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + ran = nxt_router_start_timeout_test_drain(task, &engine); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the deadline did not fire " + "for a sent-but-uncompleted start"); + goto done; + } + + if (app->proto_port_requests != 0 || app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a sent-but-uncompleted " + "start was not failed once its completion ran, " + "proto_port_requests %uD pending %uD (expected 0 and 0)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * A start whose message is only partially transmitted. + * + * nxt_port_write_handler() sets port_msg.nf once a fragment has gone out, + * and clears msg->fd[] in the same breath: the peer is mid-stream and the + * descriptors are already with it, so nxt_port_socket_cancel() must refuse + * to take the message back. The deadline must then leave the RPC armed + * -- failing it would release the payload the remaining fragments point at + * -- and fail it from the completion of the last fragment instead. This + * is the documented limitation of a protocol without cancellation. + */ + + app->pending_processes = 1; + app->use_count = 2; + + nxt_router_start_app_process_handler(task, router_port, app); + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: no message queued for the " + "partial-send leg"); + goto done; + } + + sent->port_msg.nf = 1; + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the deadline did not fire " + "for a partially sent start"); + goto done; + } + + if (nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a partially sent message was " + "taken back out of the send queue"); + goto done; + } + + if (app->proto_port_requests != 1 || app->pending_processes != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a partially sent start was " + "failed while its payload was still queued, " + "proto_port_requests %uD pending %uD (expected 1 and 1)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * The remainder completes -- here by the destination dying, which is what + * nxt_port_error_handler() does to every message it still holds. The + * failure must arrive with it and not before. + */ + + nxt_port_test_run_error_handler(task, main_port); + + (void) nxt_router_start_timeout_test_drain(task, &engine); + + if (app->proto_port_requests != 0 || app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a partially sent start was " + "not failed by its final completion, " + "proto_port_requests %uD pending %uD (expected 0 and 0)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * The other shape of START_PROCESS: a worker start, sent to a prototype + * that is already up. It carries no payload and no descriptors, so the + * deadline has nothing to wait on and fails the RPC itself -- but the + * message can still be sitting in the destination's send queue, and + * delivered after the stream is retired it starts a worker whose + * PROCESS_READY answers nobody, leaving a process and a port the router is + * not tracking. Recall it while it is still there. + * + * main_port stands in for the prototype: what matters is a destination + * that is not write-ready and has no shared-memory queue, so the message + * queues rather than going straight out. + */ + + app->proto_port = main_port; + + app->pending_processes = 1; + app->use_count = 2; + + /* Baseline for the reaper's inherited reference, checked below. */ + + joint_use = joint->use_count; + + nxt_router_start_app_process_handler(task, router_port, app); + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL || sent->buf != NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the worker start queued no " + "header-only message"); + goto done; + } + + /* Read before the recall below frees the message. */ + + stream = sent->port_msg.stream; + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded a worker " + "start"); + goto done; + } + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: an expired worker start left " + "its START_PROCESS queued, to start a worker for a " + "stream that is already retired"); + goto done; + } + + /* + * A recalled start forked nothing, so it is accounted for as any other + * failed attempt: the slot goes straight back. Moving it to + * unaccounted_processes instead would strand it -- no PROCESS_READY and + * no REMOVE_PID are coming for a process that was never created -- and + * that would make "processes": {"max"} a ratchet rather than a bound. + */ + + if (app->pending_processes != 0 || app->unaccounted_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a recalled worker start left " + "pending %uD unaccounted %uD (expected 0 and 0)", + app->pending_processes, app->unaccounted_processes); + goto done; + } + + if (joint->use_count != joint_use) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a recalled worker start left " + "joint %uD (expected %uD)", joint->use_count, joint_use); + goto done; + } + + /* + * The start that does leave a process behind: one whose START_PROCESS + * reached the prototype, which forked a worker that never announced + * itself. Taking the message out of the send queue here is what the + * prototype's read does in the field, so the deadline below finds nothing + * to recall and knows the fork happened. + * + * The router has no pid for that worker -- a pid is what PROCESS_READY + * carries -- so it can neither kill it nor count it in ->processes. The + * slot moves to unaccounted_processes, where nxt_router_app_can_start() + * still sees it, and "processes": {"max"} keeps bounding OS children. + */ + + app->pending_processes = 1; + app->use_count = 2; + + joint_use = joint->use_count; + + nxt_router_start_app_process_handler(task, router_port, app); + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL || sent->buf != NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the delivered worker start " + "queued no header-only message"); + goto done; + } + + stream = sent->port_msg.stream; + + if (nxt_port_socket_cancel(task, main_port, NXT_PORT_MSG_START_PROCESS, + stream, router_port->id, NULL) + != NXT_PORT_MSG_CANCELLED) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: could not stand in for the " + "prototype's read of the worker start"); + goto done; + } + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded a delivered " + "worker start"); + goto done; + } + + if (app->pending_processes != 0 || app->unaccounted_processes != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: an expired worker start left " + "pending %uD unaccounted %uD (expected 0 and 1)", + app->pending_processes, app->unaccounted_processes); + goto done; + } + + /* + * And it is a bound rather than a leak because the expiry put a handler + * back on the stream. This is the REMOVE_PID the router gets when that + * worker finally dies: the prototype which forked it reaps it and + * notifies with the start's own stream still attached, and + * nxt_router_remove_pid_handler() retypes that into an RPC error. The + * slot comes back, and so does the joint reference the reaper inherited. + */ + + nxt_port_rpc_error(task, router_port, stream); + + if (app->unaccounted_processes != 0 || joint->use_count != joint_use) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the death of the process an " + "expired start left behind gave back unaccounted %uD " + "joint %uD (expected 0 and %uD)", + app->unaccounted_processes, joint->use_count, joint_use); + goto done; + } + + app->proto_port = NULL; + + /* + * The other shape that leaves a process behind: a prototype start whose + * START_PROCESS reached main, which forked the prototype for it. A + * prototype that never announces itself forks no worker of its own, so + * the attempt leaves exactly one process however many requests parked on + * it -- and one slot has to account for that process, exactly as a + * worker's does. + * + * Two callers park here: the initiator, and one that joined the wait. + * The cohort stands for requests, so the joiner's slot goes straight + * back and the initiator's is the one that stays. Treating the whole + * cohort as starts that never happened, which is what the router did + * before, returned both: every later request then forked another + * prototype that would never answer either, and "processes": {"max"} + * bounded nothing at all. + */ + + app->pending_processes = 2; + app->unaccounted_processes = 0; + app->use_count = 3; + + joint_use = joint->use_count; + + nxt_router_start_app_process_handler(task, router_port, app); + nxt_router_start_app_process_handler(task, router_port, app); + + if (app->proto_port_requests != 2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the second caller did not " + "join the prototype wait, proto_port_requests %uD " + "(expected 2)", app->proto_port_requests); + goto done; + } + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent == NULL || sent->buf == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the prototype start queued " + "no payload message"); + goto done; + } + + stream = sent->port_msg.stream; + + /* + * Standing in for main's read. Consuming the payload is the half that + * matters: nxt_port_write_handler() advances the buffer as it sends, and + * that is how the completion below tells a message the destination has + * seen whole from one recalled or dropped with bytes still in it. + */ + + sent->buf->mem.pos = sent->buf->mem.free; + + if (nxt_port_socket_cancel(task, main_port, NXT_PORT_MSG_START_PROCESS, + stream, router_port->id, NULL) + != NXT_PORT_MSG_CANCELLED) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: could not stand in for " + "main's read of the prototype start"); + goto done; + } + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded a delivered " + "prototype start"); + goto done; + } + + if (app->pending_processes != 0 || app->unaccounted_processes != 1 + || app->proto_port_requests != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: an expired prototype start " + "left pending %uD unaccounted %uD proto_port_requests " + "%uD (expected 0, 1 and 0)", + app->pending_processes, app->unaccounted_processes, + app->proto_port_requests); + goto done; + } + + /* + * And the prototype's death gives the slot back, the same way a worker's + * does: main leaves the start's stream on a process that never reached + * READY, so the REMOVE_PID for it is retyped onto the reaper. + */ + + nxt_port_rpc_error(task, router_port, stream); + + if (app->unaccounted_processes != 0 || joint->use_count != joint_use) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the death of the prototype " + "an expired start left behind gave back unaccounted %uD " + "joint %uD (expected 0 and %uD)", + app->unaccounted_processes, joint->use_count, joint_use); + goto done; + } + + /* + * A start whose write is accepted and then dropped inside the port layer. + * + * With the port write-ready and its queue empty, nxt_port_msg_chk_insert() + * declines to queue and nxt_port_socket_write2() sends inline through + * nxt_port_write_handler(). A send that fails there -- here because the + * descriptor is -1, in the field because the peer is gone, or because + * nxt_port_msg_insert_tail() cannot allocate after an NXT_AGAIN, which + * happens against a perfectly live port -- drops a message that was never + * in port->messages. write2() still answers NXT_OK, because the handler + * returns void. + * + * So the deadline finds nothing to cancel and, waiting for the payload to + * be released, would wait for a completion nobody was going to run: the + * RPC and its pending_processes slot armed for good, which is the very + * wedge the option exists to bound. nxt_port_msg_drop() completes the + * buffers of these dropped messages, so the wait ends the way every other + * one does. + */ + + app->pending_processes = 1; + app->use_count = 2; + + /* + * The inline path really does call into the socket layer, which logs + * through the event's own task and sizes its iovec against the port's send + * limit. A port from nxt_port_new() has neither -- nxt_port_write_enable() + * is what fills them in for a real one -- and the descriptor stays -1, so + * the sendmsg() fails and the drop path is taken. + */ + + main_port->socket.task = task; + main_port->socket.log = thr->log; + main_port->max_size = 1024; + main_port->socket.write_ready = 1; + + nxt_router_start_app_process_handler(task, router_port, app); + + main_port->socket.write_ready = 0; + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the dropped-write leg queued " + "its message instead of sending it inline"); + goto done; + } + + /* The completion the drop queued, and the error handler it raised. */ + + (void) nxt_router_start_timeout_test_drain(task, &engine); + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded a start " + "whose write was dropped inside the port layer"); + goto done; + } + + if (app->proto_port_requests != 0 || app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a start whose write was " + "dropped stayed armed, proto_port_requests %uD pending " + "%uD (expected 0 and 0)", + app->proto_port_requests, app->pending_processes); + goto done; + } + + /* + * And the same expiry on the config-apply path, which is where the payload + * has a lifetime of its own: nxt_router_app_rpc_create() allocates it from + * tmcf->mem_pool, and the failure it drives ends in + * nxt_router_conf_error(), which releases that pool. A deadline that + * failed the RPC with the message still queued would leave the send queue + * pointing into freed memory -- which is what this leg would report under + * ASan, from the teardown that drains the port below. + */ + + nxt_memzero(&test_router, sizeof(nxt_router_t)); + nxt_queue_init(&test_router.sockets); + nxt_queue_init(&test_router.apps); + + rt->port_by_type[NXT_PROCESS_ROUTER] = router_port; + + tmcf = nxt_router_test_temp_conf(task); + if (nxt_slow_path(tmcf == NULL)) { + goto done; + } + + tmcf->router_conf->router = &test_router; + + /* nxt_router_conf_send() answers on it and drops a reference for it. */ + tmcf->port = main_port; + nxt_port_inc_use(main_port); + + nxt_router_test_app_rpc_create(task, tmcf, app); + + if (nxt_router_start_timeout_test_msg(main_port) == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the config-apply start " + "queued nothing"); + goto done; + } + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: nothing bounded the " + "config-apply start"); + goto done; + } + + /* + * tmcf and its pool are gone from here on. What must not be gone with + * them is a queued reference to the payload they held: only the RPC_ERROR + * nxt_router_conf_send() wrote may remain. + */ + + sent = nxt_router_start_timeout_test_msg(main_port); + + if (sent != NULL + && sent->port_msg.type == (NXT_PORT_MSG_START_PROCESS + & NXT_PORT_MSG_MASK)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: the config-apply start left " + "its START_PROCESS queued against a released pool"); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router start timeout test passed"); + + ret = NXT_OK; + +done: + + if (shared_port != NULL) { + nxt_port_use(task, shared_port, -1); + } + + if (main_port != NULL) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + + /* + * Both prototype requests were queued rather than written, and + * nxt_port_msg_chk_insert() takes a port reference for each. Left + * alone the port never reaches nxt_port_mp_cleanup(), so its pool and + * messages leak and the assertions there never run. + */ + nxt_port_test_run_error_handler(task, main_port); + + nxt_port_use(task, main_port, -1); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + if (timers) { + nxt_free(engine.timers.changes); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 52c4d1e22..1557c6593 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -221,6 +221,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_start_timeout_test(thr) != NXT_OK) { + return 1; + } + if (nxt_router_remove_pid_soak_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 9af54820c..bc4c7777a 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -79,6 +79,7 @@ nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_death_test(nxt_thread_t *thr); +nxt_int_t nxt_router_start_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); diff --git a/test/test_app_start_timeout.py b/test/test_app_start_timeout.py new file mode 100644 index 000000000..416341b86 --- /dev/null +++ b/test/test_app_start_timeout.py @@ -0,0 +1,570 @@ +"""A worker that starts but never announces itself must not strand the start. + +An application process is only ever announced by the process itself, from +nxt_unit_init() -> PROCESS_READY. A `"type": "external"` binary that never +gets that far -- /bin/sleep is the shortest example, but a wrapper script or a +runtime blocked in its own init reaches the same state -- answers nothing, and +because it stays alive nothing kills it either, so no REMOVE_PID converts the +wait into an error. + +Without a "limits": {"start_timeout"} that leaves the router's START_PROCESS +RPC armed for good. With the default "processes" that RPC is the only continuation +of nxt_router_conf_apply(), so: + + * the configuration PUT never returned, and + * the controller parks the in-flight request at the head of its queue, so + nxt_controller_check_postpone_request() queued every later control request + behind it -- GET /status included. + +The whole control plane was unavailable for as long as that process lived. + +The bound is opt-in: NXT_APP_START_TIMEOUT is 0, because the window it +measures is the application's own startup and a default would fail a slow but +honest one (see the comment on that macro). So these tests set it explicitly, +and test_app_start_timeout_default_is_unbounded() pins the default. + +Every test here would hang rather than fail on a build without the deadline, +which is why the module is worth running under `timeout` when comparing. +""" + +import re +import subprocess +import time + +import pytest + +from unit.control import Control +from unit.log import Log +from unit.option import option + +prerequisites = {} + +client = Control() + + +# Seconds, like every other "limits" member (NXT_CONF_MAP_MSEC). Long enough +# that a slow-but-honest start is not mistaken for a wedged one, short enough +# to keep the module quick. There is no product default to inherit: +# NXT_APP_START_TIMEOUT is 0, meaning unbounded. +START_TIMEOUT = 3 + +STUCK_ARG = 'unit-test-app-start-timeout' + + +def _stuck_conf(start_timeout=START_TIMEOUT): + app = { + 'type': 'external', + # `sh -c CMD NAME` runs CMD with $0 = NAME, which puts a marker in the + # process title that _stuck_pids() can grep for without matching a + # sleep that belongs to something else on the box. The trailing `:` + # keeps sh from exec'ing the sleep over itself, which would drop the + # marker again. + 'executable': '/bin/sh', + 'arguments': ['-c', 'sleep 600; :', STUCK_ARG], + 'limits': {'start_timeout': start_timeout}, + } + + return { + 'listeners': {'*:8080': {'pass': 'applications/stuck'}}, + 'applications': {'stuck': app}, + } + + +def _serving_conf(): + return { + 'listeners': {'*:8080': {'pass': 'routes'}}, + 'routes': [{'action': {'return': 200}}], + } + + +def _stuck_pids(): + """Pids of the workers this module started, if any are still alive.""" + + res = subprocess.run( + ['pgrep', '-f', STUCK_ARG], + capture_output=True, + text=True, + check=False, + ) + + return [int(p) for p in res.stdout.split()] + + +def _children(pid): + """Pids the kernel currently lists as children of pid.""" + + try: + with open(f'/proc/{pid}/task/{pid}/children', encoding='utf-8') as f: + return [int(p) for p in f.read().split()] + + except OSError: + return [] + + +def _reap_stuck(): + """Kill the silent workers, and the `sleep` each of them forked. + + The marker is the shell's $0, so _stuck_pids() finds the shell and never + the `sleep 600` it forked -- and killing only the shell reparents that + sleep to the nearest subreaper, where it keeps running. Nothing in this + module notices it afterwards: the pid no longer carries the marker and no + longer has a parent we know. Unit's own process group is the backstop + (conftest reaps it in unit_stop), but Unit is started once per session, so + until then one sleep per test survives into every module that follows. + + Collect the children before killing the shell: afterwards they are not its + children any more. + """ + + for pid in _stuck_pids(): + children = _children(pid) + + subprocess.run(['kill', '-9', str(pid)], check=False) + + for child in children: + subprocess.run(['kill', '-9', str(child)], check=False) + + +@pytest.fixture(autouse=True) +def _stuck_app(skip_alert): + """Every test here deliberately produces alerts; name them all once. + + * the deadline's own alert; + * "failed to apply new conf", which nxt_router_conf_error() logs for the + PUT the deadline makes fail; + * "exited on signal", from reaping the silent worker below -- the router + cannot kill it (see test_app_start_timeout_worker_not_reaped), so the + test has to, and the prototype reports the death. + """ + + skip_alert( + r'did not become ready in time', + r'failed to apply new conf', + r'exited on signal', + ) + + yield + + # The router never learns the silent worker's pid -- it arrives only with + # PROCESS_READY -- so the deadline cannot kill it; see the module note in + # test_app_start_timeout_worker_not_reaped(). Clean up regardless, so a + # failure here does not leave sleeps behind for the next run. + _reap_stuck() + + +def test_app_start_timeout_put_fails(): + assert 'success' in client.conf(_serving_conf()), 'baseline configured' + + started = time.monotonic() + + resp = client.conf(_stuck_conf()) + + elapsed = time.monotonic() - started + + # The PUT must fail, not hang and not silently succeed. + assert 'error' in resp, f'stuck app rejected, got {resp}' + assert 'Failed to apply new configuration' in resp['error'], resp + + # Bounded by the deadline, with room for the fork and the round trips. + assert elapsed < START_TIMEOUT + 10, f'PUT returned in {elapsed:.1f}s' + + # And not *before* the deadline: a start that is merely slow must not be + # failed early, so the bound has to be the configured one. + assert elapsed >= START_TIMEOUT - 1, f'PUT returned in {elapsed:.1f}s' + + +def test_app_start_timeout_control_plane_survives(): + assert 'success' in client.conf(_serving_conf()), 'baseline configured' + + assert 'error' in client.conf(_stuck_conf()), 'stuck app rejected' + + # The symptom that made this severe: with the request parked at the head + # of the controller's queue, every later control request queued behind it. + started = time.monotonic() + + status = client.conf_get('/status') + + assert time.monotonic() - started < 10, 'GET /status answers promptly' + assert 'connections' in status, status + + # The failed PUT must not have installed anything. + assert 'stuck' not in status['applications'], status['applications'] + + # And the previous configuration is still the live one, still serving. + assert client.conf_get() == _serving_conf(), 'previous config retained' + + resp = client.get(url='/') + + assert resp['status'] == 200, 'previous config still serves' + + +def test_app_start_timeout_alert(wait_for_record): + assert 'success' in client.conf(_serving_conf()), 'baseline configured' + assert 'error' in client.conf(_stuck_conf()), 'stuck app rejected' + + assert ( + wait_for_record( + r'app "stuck" process did not become ready in time', wait=50 + ) + is not None + ), 'the alert names the application' + + assert re.search( + r'start_timeout', Log.read() + ), 'the alert points at the knob that raises the bound' + + +def test_app_start_timeout_knob_validated(): + """The knob is a real "limits" member, not something the parser drops. + + Asserted against an application that is actually in the configuration: a + PUT to /config/applications//limits on an app that was never applied + is rejected with "Value doesn't exist.", which would satisfy an assertion + that only looks for an error. /bin/true announces nothing either, but it + exits at once, and a start that fails by the worker dying is the + pre-existing path -- it needs no deadline and leaves nothing behind. + """ + + conf = { + 'listeners': {'*:8080': {'pass': 'routes'}}, + 'routes': [{'action': {'return': 200}}], + 'applications': { + 'stuck': { + 'type': 'external', + 'executable': '/bin/true', + 'processes': {'spare': 0}, + } + }, + } + + assert 'success' in client.conf(conf), 'application configured' + + path = 'applications/stuck/limits' + + resp = client.conf('{"start_timeout": "soon"}', path) + + assert 'error' in resp, 'a non-integer start_timeout is rejected' + + # Rejected for the right reason. A build without the knob rejects this + # with "Unknown parameter", and an application that is not in the + # configuration is rejected with "Value doesn't exist." -- either would + # satisfy the assertion above for nothing. + detail = resp.get('detail', '') + + assert 'Unknown parameter' not in detail, resp + assert "doesn't exist" not in detail, resp + assert 'integer' in detail, resp + + # And 0 -- the default, meaning unbounded -- is accepted and survives. + assert 'success' in client.conf('{"start_timeout": 0}', path), 'zero accepted' + + assert client.conf_get(f'{path}/start_timeout') == 0, 'zero round-trips' + + +def test_app_start_timeout_range_validated(): + """The knob is bounded by what a 32-bit millisecond clock can express. + + The value reaches the router through NXT_CONF_MAP_MSEC, which computes + (nxt_msec_t) seconds * 1000 and checks nothing. Above + NXT_INT32_T_MAX / 1000 the product lands past the sign bit, where + nxt_msec_diff() reads the deadline as already past -- so a configuration + asking for ~24.9 days would instead have fired immediately, silently + turning a bound meant to be generous into one that fails every start. + Negative seconds are an out-of-range conversion to an unsigned type. + """ + + conf = { + 'listeners': {'*:8080': {'pass': 'routes'}}, + 'routes': [{'action': {'return': 200}}], + 'applications': { + 'stuck': { + 'type': 'external', + 'executable': '/bin/true', + 'processes': {'spare': 0}, + } + }, + } + + assert 'success' in client.conf(conf), 'application configured' + + path = 'applications/stuck/limits' + + # Negative. + resp = client.conf('{"start_timeout": -1}', path) + + assert 'error' in resp, f'a negative start_timeout is rejected, got {resp}' + + detail = resp.get('detail', '') + + assert 'start_timeout' in detail, resp + assert 'negative' in detail, resp + + # Above NXT_INT32_T_MAX / 1000 == 2147483. + resp = client.conf('{"start_timeout": 2147484}', path) + + assert 'error' in resp, f'an out-of-range start_timeout is rejected, {resp}' + + detail = resp.get('detail', '') + + assert 'start_timeout' in detail, resp + assert '2147483' in detail, resp + + # And the bound itself is not off by one: the largest value the clock can + # express is still accepted. + assert 'success' in client.conf( + '{"start_timeout": 2147483}', path + ), 'the maximum is accepted' + + assert client.conf_get(f'{path}/start_timeout') == 2147483 + + +def test_app_start_timeout_worker_not_reaped(): + """Documents the boundary of this fix. + + The router only ever learns a worker's pid from PROCESS_READY, which is + exactly the message a silent worker does not send, so the deadline has no + pid to kill. Reaping the process needs the *prototype* -- which does know + the pid -- to escalate a child that will not exit, in + nxt_proto_sigchld_handler()/nxt_process_quit(); that is the code #268 + rewrites, so it is deliberately left out of this change. + + This test asserts the state that fix will change, so that landing it turns + this assertion red rather than letting the gap go unnoticed. + """ + + assert 'success' in client.conf(_serving_conf()), 'baseline configured' + assert 'error' in client.conf(_stuck_conf()), 'stuck app rejected' + + # The router recovered; the process it gave up on is still there. + assert client.conf_get('/status')['connections'] is not None + + time.sleep(1) + + assert _stuck_pids() != [], ( + 'the silent worker is expected to survive the timeout; if this fails, ' + 'something now reaps it -- update this test and the note above' + ) + + +def test_app_start_timeout_default_is_unbounded(findall): + """With no "limits", a worker that is merely slow must still be waited for. + + NXT_APP_START_TIMEOUT is 0, so nothing is armed at all: the start is as + unbounded as it was before this change. Exercised with a worker that + really does take its time before announcing itself -- the libunit sample + app behind a `sleep`, which is the shape of every module's startup (user + code first, nxt_unit_init() after) -- so a default bound short enough to + matter would show up here as a failed PUT rather than as a served request. + """ + + delay = 4 + + conf = { + 'listeners': {'*:8080': {'pass': 'applications/slow'}}, + 'applications': { + 'slow': { + 'type': 'external', + 'working_directory': option.temp_dir, + # `sh -c 'sleep N; exec APP' -` announces only after the sleep; + # exec keeps the app in the process the prototype forked, and + # NXT_UNIT_INIT is inherited across it. + 'executable': '/bin/sh', + 'arguments': [ + '-c', + f'sleep {delay}; exec "$0"', + f'{option.current_dir}/build/unit_app_test', + ], + } + }, + } + + started = time.monotonic() + + assert 'success' in client.conf(conf), 'a slow start is still waited for' + + elapsed = time.monotonic() - started + + assert elapsed >= delay, f'the worker announced early ({elapsed:.1f}s)' + + assert client.get(url='/')['status'] == 200, 'the slow app serves' + + assert findall(r'did not become ready in time') == [], ( + 'no deadline fired without an explicit start_timeout' + ) + + +def _stuck_on_demand_conf(max_processes, start_timeout=START_TIMEOUT): + """The same silent worker, started by a request rather than by the PUT. + + "spare": 0 is what moves the start off the configuration-apply path: with + nothing to prefork the PUT returns at once, and the first request is what + asks for a process. That is the shape in which the deadline can fire more + than once for one application, which is what these tests are about. + """ + + conf = _stuck_conf(start_timeout) + conf['applications']['stuck']['processes'] = { + 'max': max_processes, + 'spare': 0, + } + + return conf + + +def _app_processes(name='stuck'): + return client.conf_get(f'/status/applications/{name}/processes') + + +def test_app_start_timeout_bounds_orphans(): + """"processes": {"max"} must bound the processes the deadline gives up on. + + The deadline fails the start, and the worker it gave up on keeps running: + the router has no pid for it and cannot kill it (see + test_app_start_timeout_worker_not_reaped()). Such a worker is in neither + app->processes, which only PROCESS_READY fills, nor + app->pending_processes, which the failure gives back -- so unless the + router counts it somewhere, every request forks another one and "max" + bounds nothing. Five requests against "max": 2 produced five `sleep` + processes. + + app->unaccounted_processes is that somewhere. Past the bound the + application stops forking, and a request that arrives then is answered + rather than parked: only a start can take a request back out of + ack_waiting_req, and no start is coming. + """ + + max_processes = 2 + requests = max_processes + 3 + + assert 'success' in client.conf( + _stuck_on_demand_conf(max_processes) + ), 'the on-demand config applies at once' + + assert _stuck_pids() == [], 'nothing forked before the first request' + + for i in range(requests): + started = time.monotonic() + + assert client.get(url='/')['status'] == 503, f'request {i} failed' + + elapsed = time.monotonic() - started + + if i < max_processes: + # A start really was attempted, so the deadline is what answered. + assert elapsed >= START_TIMEOUT - 1, ( + f'request {i} was failed before the deadline ({elapsed:.1f}s)' + ) + + else: + # Nothing left to start: the request must not wait for a process + # that will never be asked for, nor sit in ack_waiting_req until + # it times out. + assert elapsed < START_TIMEOUT, ( + f'request {i} waited {elapsed:.1f}s for a start that cannot ' + 'happen' + ) + + pids = _stuck_pids() + + assert len(pids) == max_processes, ( + f'{len(pids)} silent workers survive "max": {max_processes}; the ' + 'deadline is forking one per request again' + ) + + procs = _app_processes() + + assert procs['unaccounted'] == max_processes, procs + assert procs['running'] == 0 and procs['starting'] == 0, procs + + +def test_app_start_timeout_orphan_death_frees_the_slot(): + """The bound is a bound, not a one-way ratchet. + + The router does hear about the process it gave up on, once: the prototype + that forked it reaps it and notifies with the start's own stream still + attached, and nxt_router_remove_pid_handler() retypes that REMOVE_PID into + an RPC error on the handler the expiry left registered. So a slot comes + back when the process behind it dies, and the application can start again. + """ + + assert 'success' in client.conf(_stuck_on_demand_conf(1)) + + assert client.get(url='/')['status'] == 503, 'the start is bounded' + + pids = _stuck_pids() + + assert len(pids) == 1, pids + assert _app_processes()['unaccounted'] == 1 + + _reap_stuck() + + for _ in range(50): + if _app_processes()['unaccounted'] == 0: + break + + time.sleep(0.1) + + procs = _app_processes() + + assert procs['unaccounted'] == 0, ( + f'the dead process still holds its slot: {procs}' + ) + + # And the slot is usable again: this forks a second silent worker. + assert client.get(url='/')['status'] == 503, 'a start is possible again' + + assert len(_stuck_pids()) == 1, 'the freed slot started a process' + + +def test_app_start_timeout_slow_worker_adopted(): + """A worker that only just missed the deadline is kept, not thrown away. + + It announces itself on the stream the expiry left registered, and the + router moves its slot from unaccounted_processes to processes and adopts + the port. Discarding it instead would be the worse trade by far: the next + request would fork another worker that is equally slow, be failed by the + same deadline, and the application would never serve anything. + """ + + delay = START_TIMEOUT + 3 + + conf = { + 'listeners': {'*:8080': {'pass': 'applications/slow'}}, + 'applications': { + 'slow': { + 'type': 'external', + 'working_directory': option.temp_dir, + 'processes': {'max': 1, 'spare': 0}, + 'limits': {'start_timeout': START_TIMEOUT}, + 'executable': '/bin/sh', + 'arguments': [ + '-c', + f'sleep {delay}; exec "$0"', + f'{option.current_dir}/build/unit_app_test', + ], + } + }, + } + + assert 'success' in client.conf(conf), 'the slow app is configured' + + # The first request pays for the deadline: the worker is still sleeping. + assert client.get(url='/')['status'] == 503, 'the deadline fired' + + assert _app_processes('slow')['unaccounted'] == 1, _app_processes('slow') + + # It announces itself a few seconds later, and is taken on. + for _ in range(100): + if _app_processes('slow')['running'] == 1: + break + + time.sleep(0.2) + + assert _app_processes('slow') == { + 'running': 1, + 'starting': 0, + 'unaccounted': 0, + 'idle': 1, + } + + assert client.get(url='/')['status'] == 200, 'the adopted worker serves' diff --git a/test/test_status.py b/test/test_status.py index 2f58f5ccd..7e30c9541 100644 --- a/test/test_status.py +++ b/test/test_status.py @@ -195,6 +195,7 @@ def check_application(name, running, starting, idle, active): 'processes': { 'running': running, 'starting': starting, + 'unaccounted': 0, 'idle': idle, }, 'requests': {'active': active}, From 7203694366f9a1e09dcf13688b4729ddc772acc6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 03:13:17 +0200 Subject: [PATCH 113/391] application: signal a worker that never announced itself A prototype told to quit writes NXT_PORT_MSG_QUIT to each of its workers and then waits: only when nxt_proto_children is empty does nxt_proto_sigchld_handler() run nxt_process_quit() for the prototype itself. A worker that has never sent PROCESS_READY is not reading its port yet, and for one that is merely slow that costs nothing: the QUIT sits in the socket and a module worker still inside the module's own start function reads it the moment nxt_unit_init() starts reading. What is lost is the QUIT to a worker that will never read that port at all. A "type": "external" one exec'd the user's binary out of nxt_app_setup() before any of ours ran, and a module worker whose start function never returns is no better. So that QUIT lands in a socket nobody will ever read, the worker keeps running, and the prototype waits on it for good -- both of them outliving every drain there is, unitd's own exit included. That was survivable while nothing failed a start: the request that asked for the worker stayed parked, so at most one such pair existed. "limits": {"start_timeout"} removes the parking, and with it the accidental bound. On the configuration-apply path nxt_router_conf_error() discards the whole nxt_app_t, so app->unaccounted_processes -- which bounds the on-demand path -- goes with it and nothing bounds this one at all. Measured, an external application running /bin/sleep with "start_timeout": 2, five PUTs of the same rejected configuration: before 5 prototypes + 5 workers, all surviving unitd's exit after 0 Ordinary control-plane retry automation therefore exhausted host pids, and the same gap made the recovery the on-demand bound relies on -- replacing the application -- collect nothing: those workers are children of the prototype, and nxt_router_free_app() sends that prototype the QUIT it could not act on. Signal the worker instead, which is possible precisely because it never announced itself: the prototype forked it and knows its pid, where the router only ever learns one from PROCESS_READY. SIGKILL, not SIGTERM, because what such a worker does with a catchable signal is the user's binary's business and one that ignores SIGTERM is the case this exists for; nothing is lost, because a worker that has not announced itself holds no port anyone can reach, no mapped queue and no request. Not at once, though. Until one of them announces itself a worker that never will is indistinguishable from one that is simply still starting, and telling those apart is exactly what "start_timeout" is. So the QUIT still goes to every worker, and where the application declared a bound the prototype arms one of its own: SIGKILL for whatever is still silent when it expires. That deadline is strictly later than the router's, and by a whole "start_timeout", because it is armed on a QUIT the router only sends once it has itself given up on the application -- so a worker that would have announced itself within the bound the user asked for gets that whole bound over again before anything is signalled. With no "start_timeout" nothing is armed and the wait is as unbounded as the wait for an application start is by default. Killing on the state alone is what this deliberately is not: a restart replenishing a spare leaves a healthy worker in the middle of its start exactly when the prototype it belongs to is told to quit, and signalling that one kills an application that was going to come up. main already maps "limits" for the prototype it forks, so "start_timeout" joins "shm" and "requests" in nxt_common_app_limits_conf and reaches the prototype with no new message and nothing new on the wire. Read process->isolated_pid, not ->pid: under "isolation": {"namespaces": {"pid": true}} nxt_proto_process_created_handler() rewrites ->pid to the global pid and only ->isolated_pid stays valid in this namespace. It is the number waitpid() reports here, and the one nxt_port_process_ready_handler() signals for the same reason. test_app_start_timeout_retry_leaks_nothing() pins the retry case and test_app_start_timeout_reconfigure_collects_orphans() the recovery; both fail without this, with five and two survivors respectively. test_app_start_timeout_slow_worker_survives_a_quit() pins the other side, and fails with an "exited on signal" on a version that reads the state alone. test_app_start_timeout_worker_not_reaped(), which pinned the gap this closes, is replaced by the first of them. Co-Authored-By: Claude Opus 5 --- src/nxt_application.c | 134 +++++++++++++++++++++- src/nxt_application.h | 9 ++ src/nxt_main_process.c | 6 + test/test_app_start_timeout.py | 197 ++++++++++++++++++++++++++++----- 4 files changed, 313 insertions(+), 33 deletions(-) diff --git a/src/nxt_application.c b/src/nxt_application.c index ad739fa86..3115492ac 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -60,6 +60,7 @@ static void nxt_proto_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_proto_process_created_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_proto_quit_children(nxt_task_t *task); +static void nxt_proto_kill_silent(nxt_task_t *task, void *obj, void *data); static nxt_process_t *nxt_proto_process_find(nxt_task_t *task, nxt_pid_t pid); static void nxt_proto_process_add(nxt_task_t *task, nxt_process_t *process); static nxt_process_t *nxt_proto_process_remove(nxt_task_t *task, nxt_pid_t pid); @@ -83,6 +84,13 @@ static nxt_lvlhsh_t nxt_proto_processes; static nxt_queue_t nxt_proto_children; static nxt_bool_t nxt_proto_exiting; +/* + * The prototype's own deadline on a quit; see nxt_proto_quit_children(). + * File scope, so its node can never outlive its storage and + * nxt_timer_disable() is never owed. + */ +static nxt_timer_t nxt_proto_kill_timer; + static nxt_app_module_t *nxt_app; static nxt_common_app_conf_t *nxt_app_conf; @@ -877,21 +885,141 @@ nxt_proto_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Tell every worker to go, and wait for the SIGCHLD of each: only when + * nxt_proto_children is empty does nxt_proto_sigchld_handler() run + * nxt_process_quit() for the prototype itself. + * + * A port message is how a worker is told, and a worker that has never sent + * PROCESS_READY is not reading its port yet. That alone does not lose the + * message: the QUIT sits in the socket, and a module worker still inside the + * module's own start function reads it the moment nxt_unit_init() starts + * reading -- which is why an application that merely takes its time to start + * shuts down cleanly, and has to keep doing so. + * + * What is lost is the QUIT to a worker that will never read that port at + * all: a "type": "external" one exec'd the user's binary out of + * nxt_app_setup() before any of ours ran, and a module worker whose start + * function never returns is no better. Such a worker outlives the request + * that asked for it and the prototype waits on it for good, so the pair + * survives every drain there is, including unitd's own exit. Measured with + * an external application running /bin/sleep and a "limits": + * {"start_timeout"} that gives up on it: five rejected configuration PUTs + * left five prototypes and five workers alive, and replacing the + * configuration did not collect them either. + * + * The two are the same worker until one of them announces itself, and + * nothing here can tell them apart -- that is precisely the question + * "start_timeout" exists to answer. So the QUIT goes to every worker as it + * always did, and where the application declared a bound, the prototype arms + * one of its own: SIGKILL for whatever is still silent when it expires, in + * nxt_proto_kill_silent(). + * + * That deadline is strictly later than the router's, and by a whole + * "start_timeout": it is armed here, and this runs on a QUIT the router only + * sends once it has itself given up on the application. A worker that would + * have announced itself within the bound the user asked for therefore gets + * that whole bound over again before anything is signalled. + * + * With no "start_timeout" nothing is armed and this is exactly what it was, + * unbounded -- as the wait for an application start is unbounded by default. + */ + static void nxt_proto_quit_children(nxt_task_t *task) { - nxt_port_t *port; - nxt_process_t *process; - nxt_runtime_t *rt; + nxt_bool_t silent; + nxt_port_t *port; + nxt_process_t *process; + nxt_runtime_t *rt; + nxt_event_engine_t *engine; rt = task->thread->runtime; + silent = 0; + nxt_queue_each(process, &nxt_proto_children, nxt_process_t, link) { + + if (nxt_slow_path(process->state != NXT_PROCESS_STATE_READY)) { + silent = 1; + } + port = nxt_process_port_first(process); nxt_runtime_port_send_quit(task, rt, port); } nxt_queue_loop; + + if (!silent || nxt_app_conf == NULL || nxt_app_conf->start_timeout == 0) { + return; + } + + engine = task->thread->engine; + + nxt_proto_kill_timer.bias = NXT_TIMER_DEFAULT_BIAS; + nxt_proto_kill_timer.work_queue = &engine->fast_work_queue; + nxt_proto_kill_timer.handler = nxt_proto_kill_silent; + nxt_proto_kill_timer.task = &engine->task; + nxt_proto_kill_timer.log = nxt_proto_kill_timer.task->log; + + nxt_timer_add(engine, &nxt_proto_kill_timer, nxt_app_conf->start_timeout); + + nxt_debug(task, "app \"%V\" quit deadline %M ms for a worker that has not " + "announced itself", + &nxt_app_conf->name, nxt_app_conf->start_timeout); +} + + +/* + * The prototype's deadline expired: a worker it told to quit has still not + * announced itself, so it never read that QUIT and never will. + * + * Signal it, which is possible precisely because it never announced itself: + * the prototype forked it and knows its pid, where the router only ever + * learns one from PROCESS_READY. SIGKILL, not SIGTERM, because what such a + * worker does with a catchable signal is the user's binary's business and one + * that ignores SIGTERM is the case this exists for; nothing is lost, because + * a worker that has not announced itself holds no port anyone can reach, no + * mapped queue (nxt_port_process_ready_handler() maps it at PROCESS_READY) + * and no request. + * + * Read process->isolated_pid, not ->pid: under "isolation": {"namespaces": + * {"pid": true}} nxt_proto_process_created_handler() rewrites ->pid to the + * global pid and only ->isolated_pid stays valid in this namespace. It is + * the number waitpid() reports in nxt_proto_sigchld_handler(), and the one + * nxt_port_process_ready_handler() signals for the same reason. + */ + +static void +nxt_proto_kill_silent(nxt_task_t *task, void *obj, void *data) +{ + nxt_process_t *process; + + nxt_queue_each(process, &nxt_proto_children, nxt_process_t, link) { + + if (process->state == NXT_PROCESS_STATE_READY) { + continue; + } + + nxt_log(task, NXT_LOG_INFO, + "app process %PI neither announced itself nor acted on the " + "quit it was sent within \"start_timeout\"; killing it", + process->isolated_pid); + + if (nxt_fast_path(process->isolated_pid > 0) + && kill(process->isolated_pid, SIGKILL) == -1) + { + /* + * ESRCH is ordinary: the worker may already have exited with its + * SIGCHLD still pending, and that SIGCHLD drains it. + */ + nxt_log(task, (nxt_errno == ESRCH) ? NXT_LOG_INFO : NXT_LOG_ALERT, + "kill(%PI, SIGKILL) failed for an app process that never " + "announced itself %E", + process->isolated_pid, nxt_errno); + } + } + nxt_queue_loop; } diff --git a/src/nxt_application.h b/src/nxt_application.h index cb547fa96..e9c74afa2 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -131,6 +131,15 @@ struct nxt_common_app_conf_s { size_t shm_limit; uint32_t request_limit; + /* + * "limits": {"start_timeout"} in milliseconds, 0 for unbounded. The + * router arms it on the start itself (nxt_router_start_timer_arm()); + * the prototype is given the same number because it is the one process + * that can act on a worker the router gave up on -- it knows its pid. + * See nxt_proto_quit_children(). + */ + nxt_msec_t start_timeout; + nxt_fd_t shared_port_fd; nxt_fd_t shared_queue_fd; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 701215d24..52fb4ebe9 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -213,6 +213,12 @@ static nxt_conf_map_t nxt_common_app_limits_conf[] = { offsetof(nxt_common_app_conf_t, request_limit), }, + { + nxt_string("start_timeout"), + NXT_CONF_MAP_MSEC, + offsetof(nxt_common_app_conf_t, start_timeout), + }, + }; diff --git a/test/test_app_start_timeout.py b/test/test_app_start_timeout.py index 416341b86..46067ff52 100644 --- a/test/test_app_start_timeout.py +++ b/test/test_app_start_timeout.py @@ -8,8 +8,8 @@ wait into an error. Without a "limits": {"start_timeout"} that leaves the router's START_PROCESS -RPC armed for good. With the default "processes" that RPC is the only continuation -of nxt_router_conf_apply(), so: +RPC armed for good. With the default "processes" that RPC is the only +continuation of nxt_router_conf_apply(), so: * the configuration PUT never returned, and * the controller parks the in-flight request at the head of its queue, so @@ -90,6 +90,36 @@ def _stuck_pids(): return [int(p) for p in res.stdout.split()] +def _stuck_protos(): + """Pids of the prototypes started for the wedging application. + + A prototype cannot exit while a child of it is alive, so it is half of + what a leaked start leaves behind and has to be counted separately: the + marker in _stuck_pids() is the worker's, not the prototype's. + """ + + res = subprocess.run( + ['pgrep', '-f', 'unit: "stuck" prototype'], + capture_output=True, + text=True, + check=False, + ) + + return [int(p) for p in res.stdout.split()] + + +def _wait_gone(seconds=10): + """Wait for every worker and prototype of the wedging app to be gone.""" + + for _ in range(int(seconds * 10)): + if _stuck_pids() == [] and _stuck_protos() == []: + break + + time.sleep(0.1) + + return _stuck_pids(), _stuck_protos() + + def _children(pid): """Pids the kernel currently lists as children of pid.""" @@ -132,9 +162,10 @@ def _stuck_app(skip_alert): * the deadline's own alert; * "failed to apply new conf", which nxt_router_conf_error() logs for the PUT the deadline makes fail; - * "exited on signal", from reaping the silent worker below -- the router - cannot kill it (see test_app_start_timeout_worker_not_reaped), so the - test has to, and the prototype reports the death. + * "exited on signal", which the prototype logs for a silent worker: the + router has no pid for one and cannot kill it, so either the test reaps + it or the prototype does, one "start_timeout" after it is told to quit + (see nxt_proto_quit_children(), and the three tests that pin it below). """ skip_alert( @@ -145,10 +176,10 @@ def _stuck_app(skip_alert): yield - # The router never learns the silent worker's pid -- it arrives only with - # PROCESS_READY -- so the deadline cannot kill it; see the module note in - # test_app_start_timeout_worker_not_reaped(). Clean up regardless, so a - # failure here does not leave sleeps behind for the next run. + # A silent worker outlives the start that asked for it, and while its + # application is live nothing collects it -- the router never learns its + # pid. Clean up regardless, so a failure here does not leave sleeps + # behind for the next run. _reap_stuck() @@ -256,7 +287,9 @@ def test_app_start_timeout_knob_validated(): assert 'integer' in detail, resp # And 0 -- the default, meaning unbounded -- is accepted and survives. - assert 'success' in client.conf('{"start_timeout": 0}', path), 'zero accepted' + assert 'success' in client.conf( + '{"start_timeout": 0}', path + ), 'zero accepted' assert client.conf_get(f'{path}/start_timeout') == 0, 'zero round-trips' @@ -318,33 +351,46 @@ def test_app_start_timeout_range_validated(): assert client.conf_get(f'{path}/start_timeout') == 2147483 -def test_app_start_timeout_worker_not_reaped(): - """Documents the boundary of this fix. +def test_app_start_timeout_retry_leaks_nothing(): + """Rejecting the same configuration N times must cost N times nothing. - The router only ever learns a worker's pid from PROCESS_READY, which is - exactly the message a silent worker does not send, so the deadline has no - pid to kill. Reaping the process needs the *prototype* -- which does know - the pid -- to escalate a child that will not exit, in - nxt_proto_sigchld_handler()/nxt_process_quit(); that is the code #268 - rewrites, so it is deliberately left out of this change. + The deadline fails the PUT, and nxt_router_conf_error() discards the + temporary configuration -- with the nxt_app_t that + app->unaccounted_processes lives on. So there is no counter left to bound + the config-apply path with, and each retry forks a fresh prototype and a + fresh worker: ordinary control-plane retry automation exhausts host pids. - This test asserts the state that fix will change, so that landing it turns - this assertion red rather than letting the gap go unnoticed. + What bounds it is not accounting but draining. Discarding the application + sends the prototype a QUIT, and the prototype is then able to act on it, + because nxt_proto_quit_children() signals a child that has never announced + itself instead of writing a port message nothing there reads. Before + that, both processes survived every retry -- and unitd's own exit. """ + retries = 5 + assert 'success' in client.conf(_serving_conf()), 'baseline configured' - assert 'error' in client.conf(_stuck_conf()), 'stuck app rejected' - # The router recovered; the process it gave up on is still there. - assert client.conf_get('/status')['connections'] is not None + for i in range(retries): + assert 'error' in client.conf(_stuck_conf()), f'PUT {i} rejected' - time.sleep(1) + workers, protos = _wait_gone() + + assert workers == [], ( + f'{len(workers)} silent workers survive {retries} rejected PUTs; ' + 'each retry is leaking the process it forked' + ) - assert _stuck_pids() != [], ( - 'the silent worker is expected to survive the timeout; if this fails, ' - 'something now reaps it -- update this test and the note above' + assert protos == [], ( + f'{len(protos)} prototypes survive {retries} rejected PUTs; a ' + 'prototype cannot exit while a child of it is alive' ) + # And none of it cost the control plane, which is the point of the bound. + assert 'connections' in client.conf_get('/status') + + assert client.conf_get() == _serving_conf(), 'previous config retained' + def test_app_start_timeout_default_is_unbounded(findall): """With no "limits", a worker that is merely slow must still be waited for. @@ -419,9 +465,9 @@ def test_app_start_timeout_bounds_orphans(): """"processes": {"max"} must bound the processes the deadline gives up on. The deadline fails the start, and the worker it gave up on keeps running: - the router has no pid for it and cannot kill it (see - test_app_start_timeout_worker_not_reaped()). Such a worker is in neither - app->processes, which only PROCESS_READY fills, nor + the router has no pid for it and cannot kill it, and its application is + live, so nothing quits the prototype that could. Such a worker is in + neither app->processes, which only PROCESS_READY fills, nor app->pending_processes, which the failure gives back -- so unless the router counts it somewhere, every request forks another one and "max" bounds nothing. Five requests against "max": 2 produced five `sleep` @@ -568,3 +614,94 @@ def test_app_start_timeout_slow_worker_adopted(): } assert client.get(url='/')['status'] == 200, 'the adopted worker serves' + + +def test_app_start_timeout_slow_worker_survives_a_quit(findall): + """A worker that is merely slow must ride out a quit, not be signalled. + + The prototype's deadline is for a worker that will never read the QUIT it + is sent (nxt_proto_quit_children()), and until one announces itself that + worker is indistinguishable from one that is simply still starting. The + deadline is how they are told apart: a slow one reads the QUIT out of its + port the moment nxt_unit_init() starts reading, which is well inside the + second "start_timeout" it is given here. + + A restart is the shape that catches it, and is what caught it: serving a + request replenishes the spare, so the application has a worker in the + middle of its start exactly when the old prototype is told to quit. + """ + + delay = 2 + + conf = { + 'listeners': {'*:8080': {'pass': 'applications/slow'}}, + 'applications': { + 'slow': { + 'type': 'external', + 'working_directory': option.temp_dir, + 'processes': {'max': 2, 'spare': 1, 'idle_timeout': 60}, + # Comfortably longer than the start below, so nothing here + # turns on a race between the two. + 'limits': {'start_timeout': delay * 10}, + 'executable': '/bin/sh', + 'arguments': [ + '-c', + f'sleep {delay}; exec "$0"', + f'{option.current_dir}/build/unit_app_test', + ], + } + }, + } + + assert 'success' in client.conf(conf), 'the slow app is configured' + + assert client.get(url='/')['status'] == 200, 'the slow app serves' + + # Serving that request is what asks for the second process, and it is + # still sleeping when the restart below quits the prototype that forked + # it. Nothing waits for it: waiting is what this test must not do. + assert 'success' in client.conf_get('/control/applications/slow/restart') + + # Long enough for both starts to finish and for a deadline that should + # not be armed at all to have fired if it were. + time.sleep(delay * 3) + + assert findall(r'killing it') == [], 'a slow worker was signalled' + + assert findall(r'exited on signal') == [], 'a slow worker was killed' + + assert client.get(url='/')['status'] == 200, 'the restarted app serves' + + +def test_app_start_timeout_reconfigure_collects_orphans(): + """Replacing the application must collect what its deadlines gave up on. + + This is the recovery the bound relies on: a worker wedged for good holds + its "processes" slot for the life of the application, and a + reconfiguration is what ends that life. It only works if the processes + actually die. They are children of the prototype, which is the one + process that knows their pids, and nxt_router_free_app() sends that + prototype a QUIT -- but a QUIT it cannot act on left the prototype waiting + on children that answer nothing, so the whole family outlived unitd. + """ + + max_processes = 2 + + assert 'success' in client.conf(_stuck_on_demand_conf(max_processes)) + + for i in range(max_processes): + assert client.get(url='/')['status'] == 503, f'request {i} failed' + + assert len(_stuck_pids()) == max_processes, 'the workers are there' + assert len(_stuck_protos()) == 1, 'and the prototype that forked them' + + assert _app_processes()['unaccounted'] == max_processes + + assert 'success' in client.conf(_serving_conf()), 'the app is replaced' + + workers, protos = _wait_gone() + + assert workers == [], f'{len(workers)} workers survive the reconfigure' + assert protos == [], f'{len(protos)} prototypes survive the reconfigure' + + assert client.get(url='/')['status'] == 200, 'the new config serves' From 7776a124c9b425bfa24a12df5c9bd078b9ecf488 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 04:04:17 +0200 Subject: [PATCH 114/391] utf8: refuse the surrogates in nxt_utf8_decode2() Its final gate is "overlong < u && u < 0x110000". For a lead byte of 0xED the overlong floor is 0x07FF, and 0x07FF < 0xD800 < 0x110000, so the bytes "ED A0 80" decode to U+D800 and the sequence is accepted -- three bytes consumed, a surrogate returned. U+D800-U+DFFF exist only to be paired inside UTF-16 and have no UTF-8 encoding (Unicode 15.0 Sect. 3.9, D92). Accepting them means the decoder hands back a code point that cannot be re-encoded, and blesses input that every strict reader refuses: Python calls "ED A0 80" an invalid continuation byte, and a record carrying those bytes is dropped whole by a consumer that decodes before parsing. The function's own comment already claims it "returns a valid character 0x80 - 0x10FFFF", which this makes true. Nothing in the product depends on the old behaviour. Outside src/nxt_utf8.* and the tests, the whole family has two callers -- nxt_utf8_encode() and nxt_utf8_length(), both in src/nxt_conf.c -- and nxt_utf8_decode2(), casecmp(), lowcase() and is_valid() have none at all. njs does not use them either; it decodes with njs_string_decode_utf8(), which substitutes U+FFFD. nxt_utf8_is_valid() is built on this decoder and inherited the hole: it returned true for "ED A0 80", so the function whose whole job is to answer that question blessed WTF-8. It has no production caller today, which is why the gap went unnoticed for as long as it did -- and why the fix wants to be here rather than at whichever call site notices first. nxt_utf8_length() at src/nxt_conf.c:2711 counts characters to report the column of a JSON parse error. A configuration holding a raw surrogate before the error offset now yields -1 there, and nxt_conf_json_position() already leaves line and column at 0 for that, with emission guarded on "resp->line != 0". The column is lost; the error is not. src/test/nxt_utf8_test.c asserted the old behaviour: its round trip walks every value below 0x110000 and requires decode() to return what encode() wrote, surrogates included, so it encoded the bug as the contract. It now requires a refusal for that range instead. Against the previous decoder it reports: nxt_utf8_decode(0d800) returned 0d800 for a surrogate, expected a refusal Co-Authored-By: Claude Opus 5 --- src/nxt_utf8.c | 13 ++++++++++++- src/test/nxt_utf8_test.c | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/nxt_utf8.c b/src/nxt_utf8.c index d983be841..ecaacaaba 100644 --- a/src/nxt_utf8.c +++ b/src/nxt_utf8.c @@ -158,7 +158,18 @@ nxt_utf8_decode2(const u_char **start, const u_char *end) } while (n != 0); - if (overlong < u && u < 0x110000) { + /* + * Shortest form, inside the Unicode range, and not a surrogate: + * U+D800-U+DFFF exist only to be paired inside UTF-16 and have no + * UTF-8 encoding at all (Unicode 15.0 Sect. 3.9, D92), so a decoder + * that returns them hands its caller a code point that cannot be + * re-encoded -- and, for anything that then writes the bytes back + * out, output no strict reader will take. + */ + + if (overlong < u && u < 0x110000 + && !(u >= 0xD800 && u <= 0xDFFF)) + { *start = p; return u; } diff --git a/src/test/nxt_utf8_test.c b/src/test/nxt_utf8_test.c index 31e5bff94..fa36dc225 100644 --- a/src/test/nxt_utf8_test.c +++ b/src/test/nxt_utf8_test.c @@ -107,6 +107,24 @@ nxt_utf8_test(nxt_thread_t *thr) d = nxt_utf8_decode(&pp, p); + /* + * The surrogates are encodable as bytes but are not UTF-8, so the + * decoder refuses them; nxt_utf8_encode() still produces the + * three-byte form, which is what a WTF-8 producer would emit. + */ + + if (u >= 0xD800 && u <= 0xDFFF) { + + if (d != 0xFFFFFFFF) { + nxt_log_alert(thr->log, "nxt_utf8_decode(%05uxD) returned " + "%05uxD for a surrogate, expected a refusal", + u, d); + return NXT_ERROR; + } + + continue; + } + if (u != d) { nxt_log_alert(thr->log, "nxt_utf8_decode(%05uxD) failed: %05uxD", u, d); From 31fcf3e49fa9281e83bcf2f1583449cc7417556f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 04:04:17 +0200 Subject: [PATCH 115/391] router: keep the access log's values inside UTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An object-format access log serialises each value with the JSON printer, and for a request header that value is bytes the client chose: RFC 9110 Sect. 5.5 admits obs-text, so 0x80-0xFF arrive unfiltered. JSON text is UTF-8 (RFC 8259 Sect. 8.1) and nxt_conf_json_escape() escapes the quote and the backslash but copies every other byte through, so a byte that begins no valid sequence lands in the record verbatim: User-Agent: caf\xff\xfe-bad {"ua":"caf\xff\xfe-bad","status":"200"} UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 10 The record is not forgeable -- the quote and the backslash are escaped, so no member can be injected -- but a consumer that decodes before it parses drops the line, and a log whose value is being structured is worth as little rejected as misread. nxt_utf8_sanitize() copies a string into the request pool with every byte that begins no valid sequence replaced by U+FFFD, and returns the original when there is nothing to replace, so an ordinary request neither allocates nor copies. The access log calls it on the resolved value before handing it to nxt_conf_set_member_string(). The sanitising deliberately does not live in nxt_conf_json_escape(), where it would be one change instead of two. That serialiser also writes state/conf.json, answers GET /config and builds the configuration sent to each worker, and there the bytes are the operator's and must round-trip exactly. Measured with the escaper doing the replacing: PUT "share": "/srv/caf\xff/$uri" GET "share": "/srv/caf�/$uri" and the same in state/conf.json A non-UTF-8 path is legal on Linux, so that rewrite points the router at a different file after a restart, and object *keys* take the same escaper -- a header-match key would come back as a different header name. Replacing bytes is right for a log record and wrong for stored configuration; only the log calls this. The single-byte case is decided inline in both passes rather than through nxt_utf8_decode(), which is a call into another translation unit: a logged value is mostly ASCII, and paying a cross-TU call per character to learn that a byte is below 0x80 is the one cost this adds to the ordinary request. No figure is claimed for it -- the mechanism is the argument, and the logging path was not benchmarked. The replacement is per byte, not per maximal subpart (Unicode 15.0 Sect. 3.9): a truncated four-byte sequence yields up to four U+FFFD where a resync yields one. That changes how many replacement characters a reader sees, never whether the output is valid, and it holds the worst case to the three bytes of U+FFFD per input byte, which is what matters when the input is a header somebody else chose. src/test/nxt_utf8_sanitize_test.c drives nineteen strings -- bytes kept, bytes replaced, and sequences that decode to nothing: lone continuations, truncated and overlong forms, surrogates, F5-and-above -- and checks the result with a strict decoder written from Unicode 15.0 Sect. 3.9 table 3-7 rather than with nxt_utf8_is_valid(), which is the decoder the sanitiser itself calls; asserting with that would only prove the sanitiser agrees with itself. It also calls the function the way its only caller does, in place: ret = nxt_utf8_sanitize(mp, &str, &str); That aliased form was broken when this was written -- "dst" was assigned before the second pass had read "src", so the log came back as a run of NUL bytes -- and the separate-arguments test did not notice. The function now reads the input through locals captured before "dst" is touched. Refs #284 Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + src/nxt_router_access_log.c | 21 +++ src/nxt_utf8.c | 103 ++++++++++++++ src/nxt_utf8.h | 2 + src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + src/test/nxt_utf8_sanitize_test.c | 221 ++++++++++++++++++++++++++++++ 7 files changed, 353 insertions(+) create mode 100644 src/test/nxt_utf8_sanitize_test.c diff --git a/auto/sources b/auto/sources index 3c1fd2d6d..2b78eb4ed 100644 --- a/auto/sources +++ b/auto/sources @@ -168,6 +168,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_sprintf_test.c \ src/test/nxt_malloc_test.c \ src/test/nxt_utf8_test.c \ + src/test/nxt_utf8_sanitize_test.c \ src/test/nxt_rbtree1_test.c \ src/test/nxt_http_parse_test.c \ src/test/nxt_strverscmp_test.c \ diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index c88553b84..28970a86f 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -325,6 +325,27 @@ nxt_router_access_log_json(nxt_task_t *task, nxt_http_request_t *r, } } + /* + * The value is whatever the variable resolved to, and for a request + * header that is bytes the client chose: RFC 9110 Sect. 5.5 admits + * obs-text, so 0x80-0xFF reach here unfiltered. JSON text is UTF-8 + * (RFC 8259 Sect. 8.1), and nxt_conf_json_escape() escapes the quote + * and the backslash but copies every other byte through -- so an + * unencodable one leaves a record that is unforgeable and still + * unreadable, which a strict consumer drops whole. + * + * Sanitize here rather than in the serializer. The same serializer + * writes state/conf.json and answers GET /config, where the bytes are + * the operator's and have to round-trip exactly: a "share" path is + * allowed to be non-UTF-8 on Linux, and rewriting one would point the + * router at a different file after a restart. + */ + + ret = nxt_utf8_sanitize(r->mem_pool, &str, &str); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + nxt_conf_set_member_string(value, &member->name, &str, i); } diff --git a/src/nxt_utf8.c b/src/nxt_utf8.c index ecaacaaba..ce3372a00 100644 --- a/src/nxt_utf8.c +++ b/src/nxt_utf8.c @@ -267,6 +267,109 @@ nxt_utf8_length(const u_char *p, size_t len) } +/* + * Copy "src" into the pool, replacing every byte that begins no valid UTF-8 + * sequence with U+FFFD. Returns "src" itself when there is nothing to + * replace, so the ordinary path neither allocates nor copies. + * + * The replacement is per byte rather than per maximal subpart (Unicode 15.0 + * Sect. 3.9): a truncated four-byte sequence therefore yields up to four + * U+FFFD where a maximal-subpart resync yields one. That changes how many + * replacement characters a reader sees, never whether the result is valid, + * and it keeps the worst-case expansion at the three bytes of U+FFFD per + * input byte -- which matters when the input is a request header somebody + * else chose. + */ + +nxt_int_t +nxt_utf8_sanitize(nxt_mp_t *mp, nxt_str_t *dst, const nxt_str_t *src) +{ + size_t len; + u_char *d, *out; + const u_char *p, *seq, *end, *start; + + /* + * "dst" and "src" may be the same nxt_str_t -- the access log sanitizes + * a value in place -- so the input is read through locals captured here + * and "dst" is not written until both passes are done. + */ + + start = src->start; + end = start + src->length; + + p = start; + len = 0; + + while (p < end) { + + /* + * nxt_utf8_decode() is a call into another translation unit, and a + * logged value is mostly ASCII, so the single-byte case is decided + * here rather than paid for at that price once per character. + */ + + if (nxt_fast_path(*p < 0x80)) { + p++; + len++; + + continue; + } + + seq = p; + + if (nxt_utf8_decode(&seq, end) == 0xFFFFFFFF) { + len += nxt_length("\xEF\xBF\xBD"); + p++; + + continue; + } + + len += seq - p; + p = seq; + } + + if (nxt_fast_path(len == src->length)) { + *dst = *src; + + return NXT_OK; + } + + out = nxt_mp_nget(mp, len); + if (nxt_slow_path(out == NULL)) { + return NXT_ERROR; + } + + d = out; + p = start; + + while (p < end) { + + if (nxt_fast_path(*p < 0x80)) { + *d++ = *p++; + + continue; + } + + seq = p; + + if (nxt_utf8_decode(&seq, end) == 0xFFFFFFFF) { + *d++ = 0xEF; *d++ = 0xBF; *d++ = 0xBD; + p++; + + continue; + } + + d = nxt_cpymem(d, p, seq - p); + p = seq; + } + + dst->start = out; + dst->length = len; + + return NXT_OK; +} + + nxt_bool_t nxt_utf8_is_valid(const u_char *p, size_t len) { diff --git a/src/nxt_utf8.h b/src/nxt_utf8.h index 13f42e16a..c21317fda 100644 --- a/src/nxt_utf8.h +++ b/src/nxt_utf8.h @@ -17,6 +17,8 @@ NXT_EXPORT u_char *nxt_utf8_encode(u_char *p, uint32_t u); +NXT_EXPORT nxt_int_t nxt_utf8_sanitize(nxt_mp_t *mp, nxt_str_t *dst, + const nxt_str_t *src); NXT_EXPORT uint32_t nxt_utf8_decode(const u_char **start, const u_char *end); NXT_EXPORT uint32_t nxt_utf8_decode2(const u_char **start, const u_char *end); NXT_EXPORT nxt_int_t nxt_utf8_casecmp(const u_char *start1, diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 1557c6593..6bd1c5b8e 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -158,6 +158,10 @@ main(int argc, char **argv) return 1; } + if (nxt_utf8_sanitize_test(thr) != NXT_OK) { + return 1; + } + if (nxt_http_parse_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index bc4c7777a..78866e78f 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -63,6 +63,7 @@ nxt_int_t nxt_gmtime_test(nxt_thread_t *thr); nxt_int_t nxt_sprintf_test(nxt_thread_t *thr); nxt_int_t nxt_malloc_test(nxt_thread_t *thr); nxt_int_t nxt_utf8_test(nxt_thread_t *thr); +nxt_int_t nxt_utf8_sanitize_test(nxt_thread_t *thr); nxt_int_t nxt_http_parse_test(nxt_thread_t *thr); nxt_int_t nxt_strverscmp_test(nxt_thread_t *thr); nxt_int_t nxt_base64_test(nxt_thread_t *thr); diff --git a/src/test/nxt_utf8_sanitize_test.c b/src/test/nxt_utf8_sanitize_test.c new file mode 100644 index 000000000..285b40fe2 --- /dev/null +++ b/src/test/nxt_utf8_sanitize_test.c @@ -0,0 +1,221 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#include +#include "nxt_tests.h" + + +/* + * nxt_utf8_sanitize() is what keeps a JSON access log record readable when + * the value came from the request: a byte that begins no valid sequence has + * no JSON representation at all (RFC 8259 Sect. 8.1 makes JSON text UTF-8), + * so it is replaced with U+FFFD rather than escaped or copied. + * + * The strings below are the ones where a sizing pass and a copying pass can + * drift apart -- bytes kept, bytes replaced, and sequences that decode to + * nothing -- because the function walks the input twice and the second walk + * fills a buffer the first one sized. + */ + +/* + * A strict UTF-8 test written from the definition rather than from + * src/nxt_utf8.c, which is the decoder the escaper itself uses. Asserting + * with that decoder would only prove the escaper agrees with itself: its + * final gate is "overlong < u && u < 0x110000", so it accepts the surrogate + * range, and a surrogate copied into the output would satisfy an assertion + * built on it while a real consumer rejected the record. Unicode 15.0 + * Sect. 3.9, table 3-7. + */ + +static nxt_bool_t +nxt_utf8_sanitize_test_utf8(const u_char *p, size_t len) +{ + uint32_t u, min; + nxt_uint_t n; + const u_char *end; + + end = p + len; + + while (p < end) { + + if (*p < 0x80) { + p++; + continue; + } + + if (*p >= 0xF0) { + if (*p > 0xF4) { + return 0; + } + + u = *p & 0x07; + n = 3; + min = 0x10000; + + } else if (*p >= 0xE0) { + u = *p & 0x0F; + n = 2; + min = 0x800; + + } else if (*p >= 0xC2) { + u = *p & 0x1F; + n = 1; + min = 0x80; + + } else { + /* A continuation byte on its own, or an overlong C0/C1 lead. */ + return 0; + } + + if (p + n >= end) { + return 0; + } + + p++; + + while (n != 0) { + if ((*p & 0xC0) != 0x80) { + return 0; + } + + u = (u << 6) | (*p & 0x3F); + p++; + n--; + } + + /* Shortest form, in range, and not a surrogate. */ + + if (u < min || u > 0x10FFFF || (u >= 0xD800 && u <= 0xDFFF)) { + return 0; + } + } + + return 1; +} + + +static const struct { + const char *name; + const char *data; + size_t length; +} nxt_utf8_sanitize_cases[] = { + { "plain", "hello", 5 }, + { "quote", "a\"b", 3 }, + { "backslash", "a\\b", 3 }, + { "named control", "\n\r\t\b\f", 5 }, + { "numeric control", "\x01\x02\x1F", 3 }, + { "nul", "\x00", 1 }, + { "two-byte utf8", "\xC3\xA9", 2 }, + { "three-byte utf8", "\xE2\x82\xAC", 3 }, + { "four-byte utf8", "\xF0\x9F\x92\xA9", 4 }, + { "lone 0xFF", "\xFF", 1 }, + { "lone continuation", "\x80", 1 }, + { "truncated two-byte", "\xC3", 1 }, + { "truncated four-byte","\xF0\x9F", 2 }, + { "overlong", "\xC0\xAF", 2 }, + { "surrogate", "\xED\xA0\x80", 3 }, + { "above 0x10FFFF", "\xF5\x80\x80\x80", 4 }, + { "invalid then quote", "\xFF\"", 2 }, + { "invalid run", "\xFF\xFE\xFD", 3 }, + { "mixed", "a\xC3\xA9\xFF\n\"z", 7 }, +}; + + +nxt_int_t +nxt_utf8_sanitize_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_str_t src, dst, alias; + nxt_int_t ret; + nxt_uint_t i; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "utf8 sanitize test started"); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + nxt_log_alert(thr->log, "utf8 sanitize test: nxt_mp_create() failed"); + return NXT_ERROR; + } + + for (i = 0; i < nxt_nitems(nxt_utf8_sanitize_cases); i++) { + + src.start = (u_char *) nxt_utf8_sanitize_cases[i].data; + src.length = nxt_utf8_sanitize_cases[i].length; + + dst.start = NULL; + dst.length = 0; + + ret = nxt_utf8_sanitize(mp, &dst, &src); + if (ret != NXT_OK) { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" failed", + nxt_utf8_sanitize_cases[i].name); + goto fail; + } + + if (!nxt_utf8_sanitize_test_utf8(dst.start, dst.length)) { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" produced " + "invalid UTF-8: \"%*s\"", + nxt_utf8_sanitize_cases[i].name, + (size_t) dst.length, dst.start); + goto fail; + } + + /* + * The access log sanitizes in place -- nxt_utf8_sanitize(mp, &s, &s) + * -- so the aliased call is the one that matters, and it is the one + * a two-pass implementation gets wrong by writing "dst" before the + * second pass has read "src". + */ + + alias = src; + + ret = nxt_utf8_sanitize(mp, &alias, &alias); + if (ret != NXT_OK) { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" failed " + "in place", nxt_utf8_sanitize_cases[i].name); + goto fail; + } + + if (alias.length != dst.length + || memcmp(alias.start, dst.start, dst.length) != 0) + { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" in place " + "gave \"%*s\", not the \"%*s\" of the separate " + "call", nxt_utf8_sanitize_cases[i].name, + (size_t) alias.length, alias.start, + (size_t) dst.length, dst.start); + goto fail; + } + + /* + * A string that needs nothing done to it must come back as itself, + * not as a copy: the ordinary request logs no invalid bytes, and + * allocating per value per request would be paid on every one. + */ + + if (nxt_utf8_sanitize_test_utf8(src.start, src.length) + && dst.start != src.start) + { + nxt_log_alert(thr->log, "utf8 sanitize test: \"%s\" copied a " + "string that was already valid", + nxt_utf8_sanitize_cases[i].name); + goto fail; + } + } + + nxt_mp_destroy(mp); + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "utf8 sanitize test passed"); + + return NXT_OK; + +fail: + + nxt_mp_destroy(mp); + + return NXT_ERROR; +} From b247437901ad28b84f231c839ba7aeaa8476b1af Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 05:19:31 +0200 Subject: [PATCH 116/391] test: wait for the workers before deleting what they run test_app_lifecycle_restart_control[node] has been failing about 1.5% of runs since 2026-08-29, on java-25 and java-26 only, always as an ERROR at teardown on "assert not alerts": [alert] Unhandled exception in application start [alert] app process exited with code 1 Neither the label nor the timestamp on that alert is true. The string exists once in the tree, at src/nxt_java.c:421, so the process is always a java one; [node] skips on a java leg but the autouse "run" fixture still runs its finalizer, so a test that never executed reports the previous test's alert. And the stamp is the worker's fork time, not the crash: a worker still inside nxt_java_start() has not reached the event loop that advances the cached clock. The alert itself is real, and it is the harness that causes it: java.lang.IllegalArgumentException: Could not load class app : java.nio.file.NoSuchFileException: /tmp/unit-test-XXXXXXXX/java/WEB-INF/classes/app.class at nginx.unit.Context.scanClasses(Context.java:1661) _clear_conf() returns when the controller acknowledges the empty configuration, but the router quits the workers asynchronously. _clear_temp_dir() then deleted the application while a worker was still reading it. Only java loses this race, because only java spends about a second in scanClasses() between its fork and its first request; every other runtime is gone by then. Unit behaves correctly throughout -- it reports an application that really did fail to start. Wait for the processes before deleting their files, and read the log a second time afterwards. The second read is not cosmetic: teardown's own output lands after the snapshot taken at the top of the fixture, so an alert provoked here was charged to whichever test ran next. Without it this change would only relabel the flake from [node] to [java]. Co-Authored-By: Claude Opus 5 --- test/conftest.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 4cf7f8a17..60ef00a09 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -253,15 +253,28 @@ def run(request): if not option.restart: _clear_conf(log=log) + + # Workers must be gone before their files are. _clear_conf() returns + # on the controller's ack, but the router quits workers asynchronously, + # and a java one still in scanClasses() keeps reading temp_dir. + _check_processes() _clear_temp_dir() # check descriptors _check_fds(log=log) - # check processes id's and amount + if option.restart: + _check_processes() + + # Teardown logs too, after the snapshot at the top of this fixture, so + # read again or those lines are charged to the next test. Not under + # --restart: it stops unitd and may have deleted the log with temp_dir. - _check_processes() + if not option.restart: + with Log.open() as f: + log += f.read() + Log.set_pos(f.tell()) # print unit.log in case of error From 747a0b536670cadaf644ceecc11ea53d9690f66f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 10 Sep 2026 01:21:37 +0200 Subject: [PATCH 117/391] test: check process identity after the pids are refreshed #307 moved _check_processes() ahead of _clear_temp_dir(), so java workers finish reading their files before those files go away. That put it before _check_fds(), which is what refreshes the recorded router and controller pids: ps['pid'] = pid_by_name(ps['name']) # unconditional if not ps['skip']: assert ps['pid'] == ps_pid, f'same pid {name}' The respawn tests call skip_fds_check(router=True), which suppresses that "same pid" assert but not the refresh -- being able to replace the router is the point of them. _check_processes() then matched the live router against the pid recorded at setup and failed: E AssertionError: one router E assert 2 == 1 Deterministic, and it aborts teardown before _check_fds() can refresh the pid, so the error is charged to unrelated tests as well: every python leg on master has been red since #307 merged. Split the waiting loop out as _wait_for_processes(). #307's early call becomes that wait alone, and the identity asserts go back after _check_fds(), where they were before. Co-Authored-By: Claude Opus 5 (1M context) --- test/conftest.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 60ef00a09..2ece98916 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -257,15 +257,19 @@ def run(request): # Workers must be gone before their files are. _clear_conf() returns # on the controller's ack, but the router quits workers asynchronously, # and a java one still in scanClasses() keeps reading temp_dir. - _check_processes() + # Only wait here: the identity asserts belong after _check_fds(), + # which is what refreshes the router/controller pids a respawn test + # has changed. + _wait_for_processes() _clear_temp_dir() # check descriptors _check_fds(log=log) - if option.restart: - _check_processes() + # check processes id's and amount + + _check_processes() # Teardown logs too, after the snapshot at the top of this fixture, so # read again or those lines are charged to the next test. Not under @@ -769,9 +773,7 @@ def _clear_temp_dir(): time.sleep(1) -def _check_processes(): - router_pid = _fds_info['router']['pid'] - controller_pid = _fds_info['controller']['pid'] +def _wait_for_processes(): main_pid = unit_instance['pid'] for _ in range(600): @@ -792,6 +794,16 @@ def _check_processes(): time.sleep(0.1) + return out + + +def _check_processes(): + router_pid = _fds_info['router']['pid'] + controller_pid = _fds_info['controller']['pid'] + main_pid = unit_instance['pid'] + + out = _wait_for_processes() + if option.restart: assert len(out) == 0, 'all termimated' return From a42376a6e4028b3a95eca0b464fc8338e8a51821 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 10 Sep 2026 00:23:10 +0200 Subject: [PATCH 118/391] wasm: decode request fields with lossy UTF-8 replacement Previously, NxtRequestInfo::get_str() converted raw string slices with std::str::from_utf8(slice).unwrap(). Because both dev and release profiles set panic = 'abort', any unauthenticated inbound request carrying non-UTF-8 bytes (such as obs-text in HTTP headers or invalid byte sequences in the request target) triggered an unhandled panic in tokio-rt-worker, aborting the application worker process (SIGABRT) and dropping all in-flight requests with a 503. Replace the unwrap() with String::from_utf8_lossy(), returning Cow<'_, str>. Valid UTF-8 incurs zero heap allocations (borrowed &str), while invalid sequences are replaced with U+FFFD without crashing the process. Add regression test test_wasm_component_obs_text in test/test_wasm-wasi-component.py asserting that non-UTF-8 targets and header values do not crash the wasm application worker. Co-Authored-By: Claude Opus 5 (1M context) --- src/wasm-wasi-component/src/lib.rs | 25 +++++++++++++------------ test/test_wasm-wasi-component.py | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/wasm-wasi-component/src/lib.rs b/src/wasm-wasi-component/src/lib.rs index 7eb26f0bc..53c969a9b 100644 --- a/src/wasm-wasi-component/src/lib.rs +++ b/src/wasm-wasi-component/src/lib.rs @@ -3,6 +3,7 @@ use bytes::{Bytes, BytesMut}; use http_body_util::combinators::BoxBody; use http_body_util::{BodyExt, Full}; use hyper::Error; +use std::borrow::Cow; use std::ffi::{CStr, CString}; use std::mem::MaybeUninit; use std::process::exit; @@ -359,8 +360,8 @@ impl GlobalState { ) -> Result { let mut request = http::Request::builder(); - request = request.method(info.method()); - request = match info.version() { + request = request.method(info.method().as_ref()); + request = match &*info.version() { "HTTP/0.9" => request.version(http::Version::HTTP_09), "HTTP/1.0" => request.version(http::Version::HTTP_10), "HTTP/1.1" => request.version(http::Version::HTTP_11), @@ -374,14 +375,14 @@ impl GlobalState { let uri = http::Uri::builder() .scheme(if info.tls() { "https" } else { "http" }) - .authority(info.server_name()) - .path_and_query(info.target()) + .authority(info.server_name().as_ref()) + .path_and_query(info.target().as_ref()) .build() .context("failed to build URI")?; request = request.uri(uri); for (name, value) in info.fields() { - request = request.header(name, value); + request = request.header(name.as_ref(), value.as_ref()); } Ok(request) } @@ -463,7 +464,7 @@ unsafe impl Send for NxtRequestInfo {} unsafe impl Sync for NxtRequestInfo {} impl NxtRequestInfo { - fn method(&self) -> &str { + fn method(&self) -> Cow<'_, str> { unsafe { let raw = (*self.info).request; self.get_str(&(*raw).method, (*raw).method_length.into()) @@ -474,21 +475,21 @@ impl NxtRequestInfo { unsafe { (*(*self.info).request).tls != 0 } } - fn version(&self) -> &str { + fn version(&self) -> Cow<'_, str> { unsafe { let raw = (*self.info).request; self.get_str(&(*raw).version, (*raw).version_length.into()) } } - fn server_name(&self) -> &str { + fn server_name(&self) -> Cow<'_, str> { unsafe { let raw = (*self.info).request; self.get_str(&(*raw).server_name, (*raw).server_name_length.into()) } } - fn target(&self) -> &str { + fn target(&self) -> Cow<'_, str> { unsafe { let raw = (*self.info).request; self.get_str(&(*raw).target, (*raw).target_length.into()) @@ -502,7 +503,7 @@ impl NxtRequestInfo { } } - fn fields(&self) -> impl Iterator { + fn fields(&self) -> impl Iterator, Cow<'_, str>)> { unsafe { let raw = (*self.info).request; (0..(*raw).fields_count).map(move |i| { @@ -593,10 +594,10 @@ impl NxtRequestInfo { &self, ptr: &bindings::nxt_unit_sptr_t, len: u32, - ) -> &str { + ) -> Cow<'_, str> { let ptr = bindings::nxt_unit_sptr_get(ptr); let slice = std::slice::from_raw_parts(ptr, len.try_into().unwrap()); - std::str::from_utf8(slice).unwrap() + String::from_utf8_lossy(slice) } } diff --git a/test/test_wasm-wasi-component.py b/test/test_wasm-wasi-component.py index b1538287a..38fd71a87 100644 --- a/test/test_wasm-wasi-component.py +++ b/test/test_wasm-wasi-component.py @@ -39,3 +39,29 @@ def conf_component(value): resp = conf_component("") assert 'must not be empty' in resp.get('detail', ''), 'empty' + + +def test_wasm_component_obs_text(): + client.load('hello_world') + + # Non-UTF-8 / obs-text bytes in target must be lossily replaced with U+FFFD + # rather than panicking/aborting the wasm worker. + resp = client.http( + b'GET /caf\xff\xfe HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Connection: close\r\n\r\n', + raw=True, + ) + assert resp['status'] == 200 + assert resp['body'] == 'Hello' + + # Non-UTF-8 / obs-text bytes in headers must also not crash the wasm worker. + resp = client.http( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Custom: caf\xff\xfe\r\n' + b'Connection: close\r\n\r\n', + raw=True, + ) + assert resp['status'] == 200 + assert resp['body'] == 'Hello' From 62e10c59d78f5b221c8cd915f4e6fd7114e467e3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 10 Sep 2026 01:02:06 +0200 Subject: [PATCH 119/391] wasm: fail a malformed request without killing the worker `run()` handles every request with `state.handle(msg).await.expect("failed to handle request")`, and both profiles set `panic = 'abort'`, so any error out of `handle()` aborted the application worker. Two of those errors come straight from request data, via `to_request_builder()`: - `Host:` carrying obs-text. Unit's `nxt_http_validate_host()` skips any byte above `]`, so it reaches the module, and `http` rejects every byte >= 0x80 in an authority. - a target carrying `<`, `>`, a backtick, DEL or a control byte. `nxt_http_target_chars` only initialises its first 64 entries, so Unit forwards them, and `http` rejects them in a path. Both are unauthenticated one-request worker kills: thread 'tokio-rt-worker' panicked at src/lib.rs:260:45: failed to handle request: failed to build URI [alert] app process 3366095 exited on signal 6 Lossy UTF-8 decoding did not help here: replacement produces U+FFFD, which `http` still refuses in an authority. Answer these with a 400 instead. `reject()` takes `self` so the request info is always released -- returning without `nxt_unit_request_done()` would leak it and hang the client. Add test_wasm_component_unrepresentable_request covering both vectors and asserting the worker still serves afterwards. Co-Authored-By: Claude Opus 5 (1M context) --- src/wasm-wasi-component/src/lib.rs | 43 ++++++++++++++++++++++++++++-- test/test_wasm-wasi-component.py | 18 +++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/wasm-wasi-component/src/lib.rs b/src/wasm-wasi-component/src/lib.rs index 53c969a9b..3f42417ad 100644 --- a/src/wasm-wasi-component/src/lib.rs +++ b/src/wasm-wasi-component/src/lib.rs @@ -293,9 +293,27 @@ impl GlobalState { // Convert the `nxt_*` representation into the representation required // by Wasmtime's `wasi-http` implementation using the Rust `http` // crate. - let request = self.to_request_builder(&info)?; + // A request that the `http` crate refuses to represent must fail on + // its own, not take the worker with it: `run()` turns an `Err` from + // here into `.expect()`, and both profiles set `panic = 'abort'`. + // Unit forwards bytes that `http` rejects -- a `Host` holding + // obs-text, a target holding `<`, `>` or a control byte -- so this is + // reachable from an unauthenticated request. + let builder = match self.to_request_builder(&info) { + Ok(builder) => builder, + Err(e) => { + info.reject(&e); + return Ok(()); + } + }; let body = self.to_request_body(&mut info); - let request = request.body(body)?; + let request = match builder.body(body) { + Ok(request) => request, + Err(e) => { + info.reject(&e.into()); + return Ok(()); + } + }; let (sender, receiver) = tokio::sync::oneshot::channel(); @@ -581,6 +599,27 @@ impl NxtRequestInfo { } } + /// Fail this one request with a 400, leaving the worker alive. + /// + /// Takes `self` so the request info is always released: returning without + /// `request_done()` would leak it and hang the client. + fn reject(mut self, err: &anyhow::Error) { + unsafe { + let msg = format!("rejected a malformed request: {err:#}"); + if let Ok(msg) = CString::new(msg) { + bindings::nxt_unit_req_log( + self.info, + bindings::NXT_UNIT_LOG_ERR as i32, + "%s\0".as_ptr().cast(), + msg.as_ptr(), + ); + } + } + self.init_response(400, 0, 0); + self.send_response(); + self.request_done(); + } + fn request_done(self) { unsafe { bindings::nxt_unit_request_done( diff --git a/test/test_wasm-wasi-component.py b/test/test_wasm-wasi-component.py index 38fd71a87..c58827237 100644 --- a/test/test_wasm-wasi-component.py +++ b/test/test_wasm-wasi-component.py @@ -65,3 +65,21 @@ def test_wasm_component_obs_text(): ) assert resp['status'] == 200 assert resp['body'] == 'Hello' + + +def test_wasm_component_unrepresentable_request(): + client.load('hello_world') + + # Unit forwards bytes that the Rust "http" crate will not put in a URI: + # obs-text in Host, and "<" or a control byte in the target. Building the + # request then fails, and that error must fail this request alone. The + # worker used to abort on it (SIGABRT), which conftest's Log.check_alerts() + # would also catch on teardown. + for req in ( + b'GET / HTTP/1.1\r\nHost: caf\xff\r\nConnection: close\r\n\r\n', + b'GET /a Date: Wed, 9 Sep 2026 04:30:20 +0200 Subject: [PATCH 120/391] static: refuse an "index" carrying a zero byte "index" is appended to a "share" directory and the result is handed to open() as a NUL-terminated C string, so an embedded NUL truncates the name at the sink. Confirmed against a running unitd: with a file named "a" in the share directory and "share": "$uri", "index": "a\u0000b" a request for "/" returned 200 and the contents of "a". The zero byte gets in because an escaped "\u0000" survives configuration parsing -- nxt_conf_json_parse_string() decodes it through nxt_utf8_encode(), which writes a single 0x00 byte. This sink already has a guard on its other half: nxt_http_static_send() rejects a "share" whose expansion contains a NUL, with a comment naming open() as the reason. "share" is a template so it has to be checked per request; "index" is a plain configuration string, so it is refused once at configuration time instead. The same guard exists for "component" and for environment names and values -- seven fields in the validator already check for a null character, and this was the eighth that needed it. An empty "index" stays valid: test_static_index configures one and expects a 404, and that is unchanged. The validator is attached in three places: the "share" action, a PHP target's "index", and the top-level "index" a PHP application uses without targets. test_static.py covered the first; test_php_application_index_nul covers the other two, with "spare": 0 so the configuration is validated without spawning a worker. Co-Authored-By: Claude Opus 5 (1M context) --- src/nxt_conf_validation.c | 38 +++++++++++++++++++++++++ test/test_php_targets.py | 60 +++++++++++++++++++++++++++++++++++++++ test/test_static.py | 19 +++++++++++++ 3 files changed, 117 insertions(+) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 5538b1a7e..3d2992ce9 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -123,6 +123,8 @@ static nxt_int_t nxt_conf_vldt_pass(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_return(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_index(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_share(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_share_element(nxt_conf_validation_t *vldt, @@ -876,6 +878,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { }, { .name = nxt_string("index"), .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_index, }, { .name = nxt_string("types"), .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, @@ -1114,6 +1117,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { }, { .name = nxt_string("index"), .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_index, }, NXT_CONF_VLDT_END @@ -1131,6 +1135,7 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { }, { .name = nxt_string("index"), .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_index, }, NXT_CONF_VLDT_NEXT(nxt_conf_vldt_php_common_members) @@ -2444,6 +2449,39 @@ nxt_conf_vldt_return(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +/* + * "index" is appended to a "share" directory and the result is handed to + * open() as a NUL-terminated C string (nxt_http_static.c, where the name is + * assembled). An embedded NUL truncates the name there, so "a\u0000b" opens + * "a". Unlike "share" this value is not a template, so it can be refused once + * at configuration time instead of on every request. + * + * The neighbouring "share" has the same guard applied per request, and + * "component" and the environment names and values are refused the same way; + * this is the sink that was missed. + */ + +static nxt_int_t +nxt_conf_vldt_index(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + nxt_str_t str; + + nxt_conf_get_string(value, &str); + + /* memchr() on a null pointer is undefined even for a zero length. */ + + if (str.length != 0 + && nxt_slow_path(memchr(str.start, '\0', str.length) != NULL)) + { + return nxt_conf_vldt_error(vldt, "The \"index\" must not contain " + "null character."); + } + + return NXT_OK; +} + + static nxt_int_t nxt_conf_vldt_share(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) diff --git a/test/test_php_targets.py b/test/test_php_targets.py index 6085db404..0f38fea0a 100644 --- a/test/test_php_targets.py +++ b/test/test_php_targets.py @@ -98,3 +98,63 @@ def test_php_application_targets_error(): assert 'error' in client.conf_delete( 'applications/targets/default/root' ), 'root remove' + + +def test_php_application_index_nul(): + # "index" is appended to a directory and handed to open() as a + # NUL-terminated C string, so an embedded NUL truncates the name at the + # sink. This validator is shared with the static "share" action + # (test_static.py::test_static_index_nul); here it is exercised on both + # PHP schemas that attach it: a target's "index" and the top-level + # "index" used without targets. "spare": 0 validates the configuration + # without spawning a PHP worker. + root = f"{option.test_dir}/php/targets" + + def conf_targets_index(index): + return client.conf( + { + "listeners": {"*:8080": {"pass": "applications/targets"}}, + "applications": { + "targets": { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "targets": { + "default": {"index": index, "root": root}, + }, + } + }, + } + ) + + def conf_notargets_index(index): + return client.conf( + { + "listeners": {"*:8080": {"pass": "applications/notargets"}}, + "applications": { + "notargets": { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "root": root, + "index": index, + } + }, + } + ) + + resp = conf_targets_index("index\0.php") + assert ( + 'must not contain null character' in resp.get('detail', '') + ), 'targets index with null character' + + assert 'success' in conf_targets_index( + "index.php" + ), 'clean targets index still accepted' + + resp = conf_notargets_index("index\0.php") + assert ( + 'must not contain null character' in resp.get('detail', '') + ), 'notargets index with null character' + + assert 'success' in conf_notargets_index( + "index.php" + ), 'clean notargets index still accepted' diff --git a/test/test_static.py b/test/test_static.py index 94fa0852c..f7fc0fa82 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -51,6 +51,25 @@ def set_index(index): assert client.get()['status'] == 404, 'index empty' +def test_static_index_nul(temp_dir): + # "index" is appended to "share" and handed to open() as a NUL-terminated + # C string, so an embedded NUL truncated the name at the sink: an index of + # "README\0x" opened "README". "share" has been guarded against this since + # it is templated; this is the same sink reached by the other half. + assert 'error' in client.conf( + {"share": f'{temp_dir}/assets$uri', "index": "README\u0000x"}, + 'routes/0/action', + ), 'index with null character' + + # An empty index stays valid -- see test_static_index. + assert 'success' in client.conf( + {"share": f'{temp_dir}/assets$uri', "index": "README"}, + 'routes/0/action', + ), 'clean index still accepted' + + assert client.get()['body'] == 'readme', 'clean index still served' + + def test_static_index_default(): assert client.get(url='/index.html')['body'] == '0123456789', 'index' assert client.get(url='/')['body'] == '0123456789', 'index 2' From 90614984d30238038448d4d1a8ad75e0160afce7 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 10 Sep 2026 16:28:58 +0200 Subject: [PATCH 121/391] wasm: stop a failed body read from moving the buffer length nxt_unit_request_read() returns a signed count, and request_read() added it to a usize: total_bytes_read += amt as usize; if total_bytes_read >= rest.len() { break; } On an error return `amt as usize` wraps to a huge number, so the loop ends and set_len() moves the buffer length far past the allocation. The `.try_into().unwrap()` below it never caught this: the value was already a usize. There was a TODO. Treat a negative count as a failed read. Stop on a zero count as well: 0 never reaches rest.len(), so a steady 0 spun in this loop forever. Also pass the space that is actually left rather than a fixed 32 MB, and refuse a count larger than that space. The buffer is reserved from content_length, so a well behaved libunit stayed inside it, but nothing here made that true -- and libunit's own read can exceed the size it was given when a body arrives partly in memory and partly in a file. Add test_wasm_component_request_body over 1 B, 8 KB and 128 KB bodies, which covers the memory and spooled paths. It does not cover the negative or zero branch: the router delivers a body the handler can read in one call, so provoking either needs a fixture that does not exist here. Co-Authored-By: Claude Opus 5 (1M context) --- src/wasm-wasi-component/src/lib.rs | 47 +++++++++++++++++++++++------- test/test_wasm-wasi-component.py | 12 ++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/wasm-wasi-component/src/lib.rs b/src/wasm-wasi-component/src/lib.rs index 3f42417ad..d2578c3b1 100644 --- a/src/wasm-wasi-component/src/lib.rs +++ b/src/wasm-wasi-component/src/lib.rs @@ -306,7 +306,13 @@ impl GlobalState { return Ok(()); } }; - let body = self.to_request_body(&mut info); + let body = match self.to_request_body(&mut info) { + Ok(body) => body, + Err(e) => { + info.reject(&e); + return Ok(()); + } + }; let request = match builder.body(body) { Ok(request) => request, Err(e) => { @@ -408,7 +414,7 @@ impl GlobalState { fn to_request_body( &self, info: &mut NxtRequestInfo, - ) -> BoxBody { + ) -> Result> { // TODO: should convert the body into a form of `Stream` to become an // async stream of frames. The return value can represent that here // but for now this slurps up the entire body into memory and puts it @@ -418,9 +424,9 @@ impl GlobalState { // TODO: can this perform a partial read? // TODO: how to make this async at the nxt level? - info.request_read(&mut body); + info.request_read(&mut body)?; - Full::new(body.freeze()).map_err(|e| match e {}).boxed() + Ok(Full::new(body.freeze()).map_err(|e| match e {}).boxed()) } fn send_response( @@ -535,25 +541,44 @@ impl NxtRequestInfo { } } - fn request_read(&mut self, dst: &mut BytesMut) { + fn request_read(&mut self, dst: &mut BytesMut) -> Result<()> { unsafe { let rest = dst.spare_capacity_mut(); let mut total_bytes_read = 0; - loop { + while total_bytes_read < rest.len() { let amt = bindings::nxt_unit_request_read( self.info, rest.as_mut_ptr().wrapping_add(total_bytes_read).cast(), - 32 * 1024 * 1024, + rest.len() - total_bytes_read, ); - total_bytes_read += amt as usize; - if total_bytes_read >= rest.len() { + + // A read returns a signed count. `amt as usize` on a + // negative one wraps to a huge number, which ends the loop + // and then moves `set_len()` past the allocation. + if amt < 0 { + bail!("failed to read the request body"); + } + + // Nothing left to read. Without this a steady 0 spins here + // forever, because 0 never reaches `rest.len()`. + if amt == 0 { break; } + + // libunit should never return more than it was asked for, + // but this count decides `set_len()` below, so do not take + // its word for it: a wrong one here is a buffer overrun, + // not a wrong answer. + let amt = amt as usize; + if amt > rest.len() - total_bytes_read { + bail!("the request body read returned more than it was given room for"); + } + + total_bytes_read += amt; } - // TODO: handle failure when `amt` is negative - let total_bytes_read: usize = total_bytes_read.try_into().unwrap(); dst.set_len(dst.len() + total_bytes_read); } + Ok(()) } fn response_write(&mut self, data: &[u8]) { diff --git a/test/test_wasm-wasi-component.py b/test/test_wasm-wasi-component.py index c58827237..30a244f85 100644 --- a/test/test_wasm-wasi-component.py +++ b/test/test_wasm-wasi-component.py @@ -83,3 +83,15 @@ def test_wasm_component_unrepresentable_request(): # The worker is still alive and serving after both. assert client.get()['status'] == 200 + + +def test_wasm_component_request_body(): + client.load('hello_world') + + # Exercises the request_read() loop. A body larger than one read keeps + # the loop going, and the component must still answer rather than spin or + # read past the buffer. + for size in (1, 8192, 128 * 1024): + resp = client.post(body='x' * size) + assert resp['status'] == 200, size + assert resp['body'] == 'Hello', size From f49f39ca4b12b5faed0fc820d59cf8aef3704241 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 10 Sep 2026 16:33:41 +0200 Subject: [PATCH 122/391] wasm: answer a failed request instead of aborting the worker run() handled each request with state.handle(msg).await.expect("failed to handle request") so every error reaching it killed the worker, and with it every other request that worker was serving. A merged change caught the two errors it could answer early; the rest still aborted: - the four libunit response calls (response_init, response_add_field, response_send, response_write) asserted their return was 0. They fail when the port or shared memory is unusable -- a router restart, a closed port -- or on a libunit state error such as a response larger than the buffer it was given. - a guest trap arrived as task.await.unwrap(), and a dropped sender as panic!("sender of response disappeared"). - a response header name longer than 255 bytes panicked converting the length for libunit's u8 parameter. Give handle() a wrapper that finishes the request whatever happens: on Ok it calls request_done(), on Err it logs and answers. The libunit wrappers return Result now, and check_rc() turns a non-zero return into an error. Answer 400 when the request itself was at fault and 500 otherwise, carried by a BadRequest marker. A failed body read is 500, not 400: the negative count comes from read() on the spooled body, which is our I/O rather than anything the client did. Once the status line has reached the router there is no status left to send, so a failure after that point only ends the request; response_started tracks it, and is set after response_send rather than response_init because init alone fills a local buffer. Leaving early must also stop the guest. Dropping a JoinHandle detaches the task rather than cancelling it, so a request that fails after the guest was spawned would finish while the wasm invocation kept its Store. GuestTask aborts a handle it still owns when dropped, and join() takes the handle out so a task that completed is never aborted. Note abort() takes effect at the guest's next yield, so it does not interrupt a compute-bound guest. Co-Authored-By: Claude Opus 5 (1M context) --- src/wasm-wasi-component/src/lib.rs | 230 +++++++++++++++++++++-------- 1 file changed, 170 insertions(+), 60 deletions(-) diff --git a/src/wasm-wasi-component/src/lib.rs b/src/wasm-wasi-component/src/lib.rs index d2578c3b1..f817486d5 100644 --- a/src/wasm-wasi-component/src/lib.rs +++ b/src/wasm-wasi-component/src/lib.rs @@ -6,6 +6,7 @@ use hyper::Error; use std::borrow::Cow; use std::ffi::{CStr, CString}; use std::mem::MaybeUninit; +use std::os::raw::c_int; use std::process::exit; use std::ptr; use std::sync::OnceLock; @@ -179,7 +180,13 @@ unsafe extern "C" fn request_handler( // Enqueue this request to get processed by the Tokio event loop, and // otherwise immediately return. let state = GLOBAL_STATE.get().unwrap(); - state.sender.blocking_send(NxtRequestInfo { info }).unwrap(); + state + .sender + .blocking_send(NxtRequestInfo { + info, + response_started: false, + }) + .unwrap(); } struct GlobalConfig { @@ -256,14 +263,26 @@ impl GlobalState { rt.block_on(async { while let Some(msg) = receiver.recv().await { let state = GLOBAL_STATE.get().unwrap(); - tokio::task::spawn(async move { - state.handle(msg).await.expect("failed to handle request") - }); + tokio::task::spawn(async move { state.handle(msg).await }); } }); } - async fn handle(&'static self, mut info: NxtRequestInfo) -> Result<()> { + /// Handle one request, and finish it whatever happens. + /// + /// Every error below used to reach `.expect()` in `run()`, which under + /// `panic = 'abort'` killed the worker and every other request on it. + async fn handle(&'static self, mut info: NxtRequestInfo) { + match self.handle_inner(&mut info).await { + Ok(()) => info.request_done(), + Err(e) => info.fail(&e), + } + } + + async fn handle_inner( + &'static self, + info: &mut NxtRequestInfo, + ) -> Result<()> { // Create a "Store" which is the unit of per-request isolation in // Wasmtime. let data = StoreState { @@ -299,27 +318,16 @@ impl GlobalState { // Unit forwards bytes that `http` rejects -- a `Host` holding // obs-text, a target holding `<`, `>` or a control byte -- so this is // reachable from an unauthenticated request. - let builder = match self.to_request_builder(&info) { - Ok(builder) => builder, - Err(e) => { - info.reject(&e); - return Ok(()); - } - }; - let body = match self.to_request_body(&mut info) { - Ok(body) => body, - Err(e) => { - info.reject(&e); - return Ok(()); - } - }; - let request = match builder.body(body) { - Ok(request) => request, - Err(e) => { - info.reject(&e.into()); - return Ok(()); - } - }; + let builder = self.to_request_builder(info).context(BadRequest)?; + + // Not BadRequest: a negative read comes from read() on the spooled + // body failing (src/nxt_unit.c), which is our I/O, not the client's + // doing, so it answers 500. + let body = self.to_request_body(info)?; + let request = builder + .body(body) + .map_err(anyhow::Error::from) + .context(BadRequest)?; let (sender, receiver) = tokio::sync::oneshot::channel(); @@ -330,7 +338,7 @@ impl GlobalState { // writing the response when it's available. This enables wasm to // generate headers, write those below, and then compute the body // afterwards. - let task = tokio::spawn(async move { + let mut task = GuestTask(Some(tokio::spawn(async move { let req = store .data_mut() .http() @@ -344,7 +352,7 @@ impl GlobalState { .await .map_err(|e| e.context("failed to invoke wasm `handle`"))?; Ok::<_, anyhow::Error>(()) - }); + }))); // Wait for the wasm to produce the initial response. If this succeeds // then propagate that failure. If this fails then wait for the above @@ -353,27 +361,24 @@ impl GlobalState { let response = match receiver.await { Ok(response) => response.context("response generation failed")?, Err(_) => { - task.await.unwrap()?; - panic!("sender of response disappeared"); + task.join().await?; + bail!("the sender of the response disappeared"); } }; // Send the headers/status which will extract the body for the next // phase. - let body = self.send_response(&mut info, response); + let body = self.send_response(info, response)?; // Send the body, a blocking operation, over time as it becomes // available. - self.send_response_body(&mut info, body) + self.send_response_body(info, body) .await .context("failed to write response body")?; // Join on completion of the wasm task which should be done by this // point. - task.await.unwrap()?; - - // And finally signal that we're done. - info.request_done(); + task.join().await?; Ok(()) } @@ -433,7 +438,7 @@ impl GlobalState { &self, info: &mut NxtRequestInfo, response: http::Response, - ) -> T { + ) -> Result { info.init_response( response.status().as_u16(), response.headers().len().try_into().unwrap(), @@ -444,13 +449,13 @@ impl GlobalState { .sum::() .try_into() .unwrap(), - ); + )?; for (k, v) in response.headers() { - info.add_field(k.as_str().as_bytes(), v.as_bytes()); + info.add_field(k.as_str().as_bytes(), v.as_bytes())?; } - info.send_response(); + info.send_response()?; - response.into_body() + Ok(response.into_body()) } async fn send_response_body( @@ -469,7 +474,7 @@ impl GlobalState { }; match frame.data_ref() { Some(data) => { - info.response_write(&data); + info.response_write(&data)?; } None => { // TODO: what to do with trailers? @@ -479,8 +484,67 @@ impl GlobalState { } } +/// Owns the wasm task so that leaving `handle_inner()` early stops it. +/// Dropping a `JoinHandle` only detaches the task, it does not cancel it: +/// a client that disconnects mid-response fails the write, the request is +/// finished without us, and the wasm invocation would keep its `Store` and +/// burn CPU in the background. `join()` takes the handle back out, so a +/// task that ran to completion is never aborted. +struct GuestTask(Option>>); + +impl GuestTask { + /// Wait for the wasm task, reporting a cancelled or panicked one as an + /// error. `unwrap()` here would panic a second time, and under + /// `panic = 'abort'` that ends the worker. A missing handle cannot + /// happen -- both callers return right after -- and is reported the + /// same way for the same reason. + async fn join(&mut self) -> Result<()> { + match self.0.take() { + Some(task) => match task.await { + Ok(result) => result, + Err(e) => bail!("the wasm task did not finish: {e}"), + }, + None => bail!("the wasm task was already joined"), + } + } +} + +impl Drop for GuestTask { + fn drop(&mut self) { + if let Some(task) = &self.0 { + task.abort(); + } + } +} + +/// Marks an error caused by what the client sent, so it is answered 400 +/// rather than 500. +#[derive(Debug)] +struct BadRequest; + +impl std::fmt::Display for BadRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("malformed request") + } +} + +impl std::error::Error for BadRequest {} + +/// libunit reports failure as a non-zero return. Turn that into an error +/// rather than a panic: under `panic = 'abort'` a panic here would take the +/// worker down and with it every other request it is serving. +fn check_rc(rc: c_int, what: &str) -> Result<()> { + if rc != 0 { + bail!("{what} failed with {rc}"); + } + Ok(()) +} + struct NxtRequestInfo { info: *mut bindings::nxt_unit_request_info_t, + // Once the status line is out, a failure can no longer be answered with + // a status -- all that is left is to end the request. + response_started: bool, } // TODO: is this actually safe? @@ -581,18 +645,24 @@ impl NxtRequestInfo { Ok(()) } - fn response_write(&mut self, data: &[u8]) { + fn response_write(&mut self, data: &[u8]) -> Result<()> { unsafe { let rc = bindings::nxt_unit_response_write( self.info, data.as_ptr().cast(), data.len(), ); - assert_eq!(rc, 0); + check_rc(rc, "nxt_unit_response_write")?; } + Ok(()) } - fn init_response(&mut self, status: u16, headers: u32, headers_size: u32) { + fn init_response( + &mut self, + status: u16, + headers: u32, + headers_size: u32, + ) -> Result<()> { unsafe { let rc = bindings::nxt_unit_response_init( self.info, @@ -600,37 +670,71 @@ impl NxtRequestInfo { headers, headers_size, ); - assert_eq!(rc, 0); + check_rc(rc, "nxt_unit_response_init")?; } + Ok(()) } - fn add_field(&mut self, key: &[u8], val: &[u8]) { + fn add_field(&mut self, key: &[u8], val: &[u8]) -> Result<()> { + // libunit takes the name length as a u8. A guest is free to emit a + // longer one, and `unwrap()` here would abort the worker over a + // header -- the failure this whole path exists to stop. + let Ok(key_len) = key.len().try_into() else { + bail!("a response header name is longer than 255 bytes"); + }; + let Ok(val_len) = val.len().try_into() else { + bail!("a response header value is too long"); + }; + unsafe { let rc = bindings::nxt_unit_response_add_field( self.info, key.as_ptr().cast(), - key.len().try_into().unwrap(), + key_len, val.as_ptr().cast(), - val.len().try_into().unwrap(), + val_len, ); - assert_eq!(rc, 0); + check_rc(rc, "nxt_unit_response_add_field")?; } + Ok(()) } - fn send_response(&mut self) { + fn send_response(&mut self) -> Result<()> { unsafe { let rc = bindings::nxt_unit_response_send(self.info); - assert_eq!(rc, 0); + check_rc(rc, "nxt_unit_response_send")?; } + // Only now has a status line reached the router. `response_init` + // alone fills a local buffer, and libunit permits a second one, so + // a failure before this point can still be answered. + self.response_started = true; + Ok(()) } - /// Fail this one request with a 400, leaving the worker alive. + /// Fail this one request, leaving the worker alive. /// - /// Takes `self` so the request info is always released: returning without - /// `request_done()` would leak it and hang the client. - fn reject(mut self, err: &anyhow::Error) { + /// Answers 500 if nothing has gone out yet. Once the status line is + /// sent there is no status left to send, so all this can do is end the + /// request and let the client see a truncated response. + fn fail(mut self, err: &anyhow::Error) { + let bad = err.downcast_ref::().is_some(); + let status = if bad { 400 } else { 500 }; + + self.log_err(&format!("failed to handle a request: {err:#}")); + + if !self.response_started + && self.init_response(status, 0, 0).is_ok() + && self.send_response().is_ok() + { + self.request_done(); + return; + } + + self.request_failed(); + } + + fn log_err(&self, msg: &str) { unsafe { - let msg = format!("rejected a malformed request: {err:#}"); if let Ok(msg) = CString::new(msg) { bindings::nxt_unit_req_log( self.info, @@ -640,9 +744,15 @@ impl NxtRequestInfo { ); } } - self.init_response(400, 0, 0); - self.send_response(); - self.request_done(); + } + + fn request_failed(self) { + unsafe { + bindings::nxt_unit_request_done( + self.info, + bindings::NXT_UNIT_ERROR as i32, + ); + } } fn request_done(self) { From 4ac60ed4ec1943db2af55b86738a56efbeed6745 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 12 Sep 2026 19:33:01 +0200 Subject: [PATCH 123/391] libunit: drop the declaration of a function that does not exist nxt_unit_split_host() is declared in the public header and defined nowhere. It appears exactly once in the tree -- that declaration -- so the first caller gets a link error rather than a working function. Upstream nginx/unit is in the same state: one occurrence, no definition. The declaration is older than this clone can show, so it has been unlinkable in both projects for as long as either history here reaches. An exported name that cannot be linked is worse than a missing one for anyone writing against libunit, because the header is the documentation. Co-Authored-By: Claude Opus 5 (1M context) --- src/nxt_unit.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/nxt_unit.h b/src/nxt_unit.h index a50046a52..46d6f3c6b 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -245,10 +245,6 @@ void nxt_unit_port_id_init(nxt_unit_port_id_t *port_id, pid_t pid, uint16_t id); /* Calculates hash for given field name. */ uint16_t nxt_unit_field_hash(const char* name, size_t name_length); -/* Split host for server name and port. */ -void nxt_unit_split_host(char *host_start, uint32_t host_length, - char **name, uint32_t *name_length, char **port, uint32_t *port_length); - /* Group duplicate fields for easy enumeration. */ void nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req); From 74560aa2598a525af755e1559242c69f90e8ad3d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 05:20:32 +0200 Subject: [PATCH 124/391] test: check for MSG_CTRUNC at runtime instead of by platform The C test suite fails on big-endian. Under qemu-s390x: port ctrunc test: credential kept, SCM_RIGHTS dropped did not truncate (controllen 48, received 48) The last two cases in nxt_port_ctrunc_test check how the kernel splits a control buffer that is too small. Both need the kernel to set MSG_CTRUNC. They were skipped with "#if (NXT_MACOSX)", because macOS does not set it. qemu is a second case of the same thing. A small test program, with no Unit code in it, reports "used=32 MSG_CTRUNC=SET" on native amd64, and "used=48 MSG_CTRUNC=NOT SET" under qemu-s390x. 48 is the value the test prints. struct ucred is 12 bytes on both platforms, so CMSG_SPACE(12) is 32 and the budget is 48 on both. The buffer arithmetic is the same and nxt_socket_msg.c behaves the same. The two cases were testing the environment, not the code. Whether the kernel sets MSG_CTRUNC is a runtime property, so check it at runtime. nxt_port_ctrunc_test_probe() sends two descriptors over a socketpair and receives them into a given budget. It uses only SCM_RIGHTS, so it works the same on every platform. It closes any descriptor the kernel installed, because the two cases it guards count open descriptors. It runs twice, once per decision the cases rely on: CMSG_LEN(0), where no descriptor fits and the kernel drops the whole cmsg, and CMSG_LEN(0) + sizeof(int), where one of two fits and the kernel installs what it can. A layer that mistranslates one need not mistranslate the other, and a predicate narrower than what it gates would let a case through to fail for the environment again. Under qemu-s390x both come back without the flag: no room: used 16 flags 0x0000; one of two: used 20 flags 0x0000 Those numbers are in the skip notice, because they are what someone debugging a future skip will want. Nothing in the probe trusts a length the kernel returned without bounding it first. msg_controllen is clamped to the budget that was offered and the descriptor walk stops at the end of the buffer as well as at cmsg_len; the whole reason the probe exists is that the layer underneath may be wrong, and an over-long cmsg_len would otherwise walk off the union and close whatever integer it found. Descriptors at or below STDERR_FILENO are a misread rather than something to close. A probe that cannot run is not a kernel that does not truncate: socketpair(), pipe(), a short sendmsg() or a failed recvmsg() return NXT_ERROR, which fails the test rather than quietly reducing its coverage. This covers macOS as well, so NXT_MACOSX is no longer used in the file. The "#if" around nxt_port_ctrunc_test_truncated() is removed with it, because the call is now decided at runtime and the function must exist everywhere. Tested on amd64: no skip message, both cases run, 48 tests, exit 0. Tested under qemu-s390x: cases skipped, 46 tests, exit 0. It exited 1 before. Refs #291 Co-Authored-By: Claude Opus 5 --- src/test/nxt_port_ctrunc_test.c | 211 ++++++++++++++++++++++++++++---- 1 file changed, 190 insertions(+), 21 deletions(-) diff --git a/src/test/nxt_port_ctrunc_test.c b/src/test/nxt_port_ctrunc_test.c index f34a8c0b0..ec7da95a9 100644 --- a/src/test/nxt_port_ctrunc_test.c +++ b/src/test/nxt_port_ctrunc_test.c @@ -179,12 +179,155 @@ nxt_port_ctrunc_test_close_fds(nxt_fd_t *fd) /* - * Defined only where it is called: both call sites sit in the non-macOS arm - * below, and an unused static trips -Wunused-function under -Werror on a - * platform no CI leg builds. + * Do "controllen" bytes of control buffer hold the two descriptors that were + * sent, and does the kernel say so? + * + * Returns NXT_OK when MSG_CTRUNC came back, NXT_DECLINED when the exchange + * worked and the flag did not, and NXT_ERROR when the probe itself could not + * run -- a caller that cannot tell those apart reports a kernel property it + * never observed. "used" and "flags" carry what was seen, for the log. + * + * Only SCM_RIGHTS is used, so this is the same on every platform that has any + * of it at all. */ -#if !(NXT_MACOSX) +static nxt_int_t +nxt_port_ctrunc_test_probe(size_t controllen, size_t *used, int *flags) +{ + int fd[2], received; + char buf, *data, *limit; + size_t i; + ssize_t n; + nxt_int_t ret; + nxt_socket_t pair[2]; + struct iovec iov; + struct msghdr msg; + struct cmsghdr *cmsg; + union { + struct cmsghdr align; + char buf[CMSG_SPACE(2 * sizeof(int))]; + } send_control; + union { + struct cmsghdr align; + char buf[CMSG_SPACE(2 * sizeof(int))]; + } recv_control; + + *used = 0; + *flags = 0; + + if (nxt_slow_path(controllen > sizeof(recv_control.buf))) { + return NXT_ERROR; + } + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_STREAM, 0, pair) != 0)) { + return NXT_ERROR; + } + + if (nxt_slow_path(pipe(fd) != 0)) { + close(pair[0]); + close(pair[1]); + + return NXT_ERROR; + } + + buf = 'x'; + iov.iov_base = &buf; + iov.iov_len = 1; + + nxt_memzero(&msg, sizeof(struct msghdr)); + nxt_memzero(&send_control, sizeof(send_control)); + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = send_control.buf; + msg.msg_controllen = sizeof(send_control.buf); + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(2 * sizeof(int)); + nxt_memcpy(CMSG_DATA(cmsg), fd, 2 * sizeof(int)); + + ret = NXT_ERROR; + + if (nxt_fast_path(sendmsg(pair[0], &msg, 0) == 1)) { + + nxt_memzero(&msg, sizeof(struct msghdr)); + nxt_memzero(&recv_control, sizeof(recv_control)); + + iov.iov_base = &buf; + iov.iov_len = 1; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = recv_control.buf; + msg.msg_controllen = controllen; + + n = recvmsg(pair[1], &msg, 0); + + if (nxt_fast_path(n == 1)) { + + /* + * Everything below walks what the kernel returned, and the reason + * this probe exists is that the layer underneath may not be one. + * Nothing here trusts a length it did not bound first: a + * msg_controllen larger than the buffer that was offered, or a + * cmsg_len reaching past it, would otherwise walk off the union + * and close whatever integer it found there. + */ + + if (msg.msg_controllen > controllen) { + msg.msg_controllen = controllen; + } + + *used = msg.msg_controllen; + *flags = msg.msg_flags; + + ret = (msg.msg_flags & MSG_CTRUNC) ? NXT_OK : NXT_DECLINED; + + limit = recv_control.buf + msg.msg_controllen; + + for (cmsg = CMSG_FIRSTHDR(&msg); + cmsg != NULL; + cmsg = CMSG_NXTHDR(&msg, cmsg)) + { + if (cmsg->cmsg_level != SOL_SOCKET + || cmsg->cmsg_type != SCM_RIGHTS) + { + continue; + } + + /* + * Whatever did fit was installed and belongs to this process + * now; leaving it open would move the descriptor count the + * cases this gates are measuring. Standard streams are never + * what was sent, so a 0, 1 or 2 here is a misread rather than + * a descriptor to close. + */ + + data = (char *) CMSG_DATA(cmsg); + + for (i = 0; data + i + sizeof(int) <= limit + && CMSG_LEN(i + sizeof(int)) <= cmsg->cmsg_len; + i += sizeof(int)) + { + nxt_memcpy(&received, data + i, sizeof(int)); + + if (received > STDERR_FILENO) { + close(received); + } + } + } + } + } + + close(fd[0]); + close(fd[1]); + close(pair[0]); + close(pair[1]); + + return ret; +} + /* * The regression itself: a control buffer that holds the credential but not @@ -280,8 +423,6 @@ nxt_port_ctrunc_test_truncated(nxt_thread_t *thr, nxt_socket_t *pair, return NXT_OK; } -#endif - /* * The control case: a buffer of the size every production receive path uses. @@ -891,22 +1032,50 @@ nxt_port_ctrunc_test(nxt_thread_t *thr) } /* - * The cases below assert how the kernel itself divides a control buffer - * that is too small: Linux and the BSDs run scm_detach_fds() against the - * room that is left and raise MSG_CTRUNC, while xnu externalizes the - * rights before it measures, so the same short receive there does not - * report truncation at all. Skipped rather than adapted: what they add - * over nxt_port_ctrunc_test_marked() is the proof that a real kernel - * produces the flag, and on a kernel that does not, there is nothing to - * prove. The code under test is platform independent -- nothing acts on - * a truncation that was not reported. + * The two cases below assert how the kernel itself divides a control + * buffer that is too small. They are two different decisions: with no + * room for any descriptor it drops the whole SCM_RIGHTS cmsg, and with + * room for one of two it installs what fits. Both must raise MSG_CTRUNC + * for the cases to mean anything, and not every layer does -- xnu + * externalizes the rights before it measures, and qemu-user's cmsg + * translation reproduces neither kernel. Nothing in the code under test + * acts on a truncation that was not reported, so where the flag does not + * come back there is nothing to prove and asserting it anyway fails for + * the environment rather than for the code. + * + * Probe both shapes rather than one: a layer that mistranslates the + * budget for the second decision need not mistranslate the first, and a + * predicate narrower than what it gates would let a case through to fail + * for the environment again. */ -#if (NXT_MACOSX) - - nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test: kernel driven " - "truncation cases skipped on macOS"); + if (ret == NXT_OK) { + nxt_int_t drop, partial; + size_t drop_used, partial_used; + int drop_flags, partial_flags; + + drop = nxt_port_ctrunc_test_probe(CMSG_LEN(0), &drop_used, + &drop_flags); + partial = nxt_port_ctrunc_test_probe(CMSG_LEN(0) + sizeof(int), + &partial_used, &partial_flags); + + if (drop == NXT_ERROR || partial == NXT_ERROR) { + nxt_log_alert(thr->log, "port ctrunc test: the MSG_CTRUNC probe " + "could not run, so the kernel driven cases cannot " + "be decided either way"); + ret = NXT_ERROR; + goto done; + } -#else + if (drop != NXT_OK || partial != NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port ctrunc test: " + "kernel driven truncation cases skipped -- this " + "kernel does not report MSG_CTRUNC (no room: used " + "%uz flags 0x%04Xd; one of two: used %uz flags " + "0x%04Xd)", drop_used, (unsigned) drop_flags, + partial_used, (unsigned) partial_flags); + goto done; + } + } if (ret == NXT_OK) { ret = nxt_port_ctrunc_test_truncated(thr, pair, @@ -932,7 +1101,7 @@ nxt_port_ctrunc_test(nxt_thread_t *thr) "one of two descriptors delivered"); } -#endif +done: nxt_socket_close(task, pair[0]); nxt_socket_close(task, pair[1]); From 7234df62767e212d29f2aa4f80cd17894554ba1c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 04:23:57 +0200 Subject: [PATCH 125/391] conf: refuse a configuration that is not UTF-8 RFC 8259 Sect. 8.1 defines JSON text as UTF-8, and the controller serves its responses as "application/json", a media type with no charset parameter. nxt_conf_json_parse_string() did not enforce it: the literal byte path is "if (ch >= ' ') continue", so every byte from 0x80 to 0xFF was stored and echoed back verbatim. A value of "caf\xff\xfe" therefore survived a PUT, was written to state/conf.json, and came back out of GET /config as bytes that no JSON parser can decode -- including the two clients this project ships: unitctl parses that response with serde_json, and the test suite with json.loads. Both require UTF-8, so a configuration unit accepted and stored could not be read back by its own tooling. The check walks the parsed tree rather than the schema. A configuration option added later cannot escape it by forgetting to ask, which is the property that matters here: this fork's zero-byte guards were written one sink at a time, and the sink nobody reported kept its hole. It runs before nxt_conf_validate(), which quotes an offending member name into its own error text. A name that is not UTF-8 would otherwise reach the response body and make the error unreadable for the very reason it is being reported. For the same reason a bad member name is reported against the object that holds it, so the pointer echoed back stays readable. A state file written before this existed still loads, exactly as written, with one warning naming the pointer. Rejecting it would drop the whole configuration on the next restart and bring the daemon back serving nothing; repairing it would silently rewrite an operator's value, which is how a working non-UTF-8 "share" path becomes a broken one. The control API refuses to store any more of them, so it is self-limiting. A zero byte is deliberately not rejected. It is valid UTF-8, and "location" accepts one today and percent-encodes it, so refusing it here would break a documented behaviour to fix a different problem. The path sinks keep their own guards. This is the first production caller of nxt_utf8_is_valid(), which until 7776a124 blessed UTF-16 surrogates and had no caller to notice. With that fix in place a raw "ED A0 80" in a configuration string is refused here too, not just the "\ud800" spelling the parser already rejected. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RqWagVeqR5uyNaDfUx6nRR --- src/nxt_conf.h | 8 +++ src/nxt_conf_validation.c | 128 +++++++++++++++++++++++++++++++++++++ src/nxt_controller.c | 55 +++++++++++++++- test/test_configuration.py | 56 ++++++++++++++++ 4 files changed, 245 insertions(+), 2 deletions(-) diff --git a/src/nxt_conf.h b/src/nxt_conf.h index 60a2061bc..a69ab7bad 100644 --- a/src/nxt_conf.h +++ b/src/nxt_conf.h @@ -128,6 +128,14 @@ void nxt_conf_json_position(u_char *start, const u_char *pos, nxt_uint_t *line, nxt_int_t nxt_conf_validate(nxt_conf_validation_t *vldt); +/* + * Checks every string in the tree -- values and member names alike -- for + * valid UTF-8. Separate from nxt_conf_validate() because + * the two callers want different answers: the control API refuses such a + * configuration, while a state file that already holds one still has to load. + */ +nxt_int_t nxt_conf_validate_encoding(nxt_conf_validation_t *vldt); + NXT_EXPORT void nxt_conf_get_string(const nxt_conf_value_t *value, nxt_str_t *str); NXT_EXPORT nxt_str_t *nxt_conf_get_string_dup(const nxt_conf_value_t *value, diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 3d2992ce9..7c8ce63b8 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -1809,6 +1809,134 @@ nxt_conf_vldt_type(nxt_conf_validation_t *vldt, const nxt_str_t *name, * * Never fails validation: on allocation failure the pointer is left empty. */ +/* + * A configuration string is JSON text, and RFC 8259 Sect. 8.1 defines JSON + * text as UTF-8. The parser does not enforce it: nxt_conf_json_parse_string() + * copies every byte above 0x1F through untouched, so a PUT can store bytes that + * no JSON parser reads back -- including the body of GET /config, which is + * served as "application/json", a media type that carries no charset parameter. + * + * This walks the parsed tree rather than the schema. A configuration option + * added later cannot escape the check by forgetting to ask for it, which is the + * property that matters here: the zero-byte guards this fork already carries + * were written one sink at a time, and the sink nobody reported kept its hole. + * + * A zero byte is deliberately NOT rejected. It is valid UTF-8, and "location" + * accepts one today and percent-encodes it (test/test_return.py:137), so + * refusing it here would break a documented behaviour to fix a different + * problem. The path sinks guard themselves; see nxt_http_static.c. + */ + +static nxt_int_t +nxt_conf_vldt_encoding_str(nxt_conf_validation_t *vldt, const nxt_str_t *str, + const char *what) +{ + if (nxt_slow_path(!nxt_utf8_is_valid(str->start, str->length))) { + return nxt_conf_vldt_error(vldt, "The %s is not valid UTF-8. JSON " + "text is UTF-8 (RFC 8259 Sect. 8.1), so " + "these bytes have no JSON representation " + "and could not be read back.", what); + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_conf_vldt_encoding_value(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value) +{ + u_char *p; + uint32_t index, next, count; + nxt_str_t name, str; + nxt_int_t ret; + nxt_conf_value_t *member; + nxt_conf_vldt_path_t seg; + + u_char buf[sizeof("4294967295") - 1]; + + switch (nxt_conf_type(value)) { + + case NXT_CONF_STRING: + nxt_conf_get_string(value, &str); + + return nxt_conf_vldt_encoding_str(vldt, &str, "value"); + + case NXT_CONF_ARRAY: + count = nxt_conf_array_elements_count(value); + + for (index = 0; index < count; index++) { + p = nxt_sprintf(buf, buf + sizeof(buf), "%uD", index); + + seg.prev = vldt->path; + seg.seg.start = buf; + seg.seg.length = p - buf; + vldt->path = &seg; + + ret = nxt_conf_vldt_encoding_value(vldt, + nxt_conf_get_array_element(value, index)); + + vldt->path = seg.prev; + + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + } + + return NXT_OK; + + case NXT_CONF_OBJECT: + next = 0; + + for ( ;; ) { + member = nxt_conf_next_object_member(value, &name, &next); + + if (member == NULL) { + break; + } + + /* + * Check the name before it becomes a path segment. The pointer + * rendered for an error is echoed in the response body, so putting + * the offending bytes there would make the error report itself + * unreadable for the same reason it is being reported. A bad name + * is reported against the object that holds it. + */ + + ret = nxt_conf_vldt_encoding_str(vldt, &name, "member name"); + + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + + seg.prev = vldt->path; + seg.seg = name; + vldt->path = &seg; + + ret = nxt_conf_vldt_encoding_value(vldt, member); + + vldt->path = seg.prev; + + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + } + + return NXT_OK; + + default: + return NXT_OK; + } +} + + +nxt_int_t +nxt_conf_validate_encoding(nxt_conf_validation_t *vldt) +{ + return nxt_conf_vldt_encoding_value(vldt, vldt->conf); +} + + static void nxt_conf_vldt_render_pointer(nxt_conf_validation_t *vldt) { diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 2f398cc9d..690ba6185 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -379,6 +379,27 @@ nxt_controller_start(nxt_task_t *task, nxt_process_data_t *data) vldt.conf_pool = mp; vldt.ver = nxt_conf_ver; + /* + * A state file written before this check existed can hold bytes the + * control API would now refuse. Rejecting it here would drop the whole + * configuration on the next restart -- the daemon would come back serving + * nothing -- and repairing it would silently rewrite an operator's value, + * which is how a working non-UTF-8 "share" path becomes a broken one. So + * it is loaded exactly as written and reported, once, with the pointer + * that names it. The API refuses to store any more of them. + */ + + if (nxt_conf_validate_encoding(&vldt) == NXT_DECLINED) { + nxt_log(task, NXT_LOG_WARN, "the restored configuration holds a value " + "at \"%V\" that the control API would now reject: %V It is " + "kept as written, and the configuration is running; correct it " + "to be able to update the configuration.", + &vldt.pointer, &vldt.error); + + nxt_memzero(&vldt.error, sizeof(nxt_str_t)); + nxt_memzero(&vldt.pointer, sizeof(nxt_str_t)); + } + ret = nxt_conf_validate(&vldt); if (nxt_slow_path(ret != NXT_OK)) { @@ -1510,7 +1531,22 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, vldt.conf_pool = mp; vldt.ver = NXT_VERNUM; - rc = nxt_conf_validate(&vldt); + /* + * Before nxt_conf_validate(), which quotes an offending member name + * into its own error text: a name that is not UTF-8 would otherwise + * reach the response body and make the error report unreadable for + * the very reason it is being reported. + * + * This runs on the tree that would be installed, so a configuration + * that already holds such a value has to have it corrected before any + * other part of it can be updated. The pointer in the error names it. + */ + + rc = nxt_conf_validate_encoding(&vldt); + + if (nxt_fast_path(rc == NXT_OK)) { + rc = nxt_conf_validate(&vldt); + } if (nxt_slow_path(rc != NXT_OK)) { nxt_mp_destroy(mp); @@ -1595,7 +1631,22 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, vldt.conf_pool = mp; vldt.ver = NXT_VERNUM; - rc = nxt_conf_validate(&vldt); + /* + * Before nxt_conf_validate(), which quotes an offending member name + * into its own error text: a name that is not UTF-8 would otherwise + * reach the response body and make the error report unreadable for + * the very reason it is being reported. + * + * This runs on the tree that would be installed, so a configuration + * that already holds such a value has to have it corrected before any + * other part of it can be updated. The pointer in the error names it. + */ + + rc = nxt_conf_validate_encoding(&vldt); + + if (nxt_fast_path(rc == NXT_OK)) { + rc = nxt_conf_validate(&vldt); + } if (nxt_slow_path(rc != NXT_OK)) { nxt_mp_destroy(mp); diff --git a/test/test_configuration.py b/test/test_configuration.py index 8342cbf14..e490b32d9 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -55,6 +55,62 @@ def test_json_unicode(): }, 'unicode get' +def test_json_utf8_invalid_value(): + # A configuration string is JSON text and JSON text is UTF-8 (RFC 8259 + # Sect. 8.1). Bytes that begin no valid sequence used to be stored and + # echoed back, which made GET /config undecodable. + resp = client.conf( + b'{"listeners": {"*:8080": {"pass": "routes"}},' + b' "routes": [{"match": {"headers": {"X-T": "caf\xff\xfe"}},' + b' "action": {"return": 200}}]}' + ) + + assert 'error' in resp, 'invalid utf-8 value rejected' + assert 'UTF-8' in resp['detail'], 'reason given' + assert ( + resp['location']['path'] == '/routes/0/match/headers/X-T' + ), 'pointer names the value' + + +def test_json_utf8_invalid_member_name(): + resp = client.conf( + b'{"listeners": {"*:8080": {"pass": "routes"}},' + b' "routes": [{"match": {"headers": {"X-\xff": "v"}},' + b' "action": {"return": 200}}]}' + ) + + assert 'error' in resp, 'invalid utf-8 member name rejected' + assert 'member name' in resp['detail'], 'name named as the culprit' + + # The pointer is echoed in this very response, so it must point at the + # object holding the bad name rather than embed the name itself -- + # otherwise the error report is unreadable for the reason it reports. + assert ( + resp['location']['path'] == '/routes/0/match/headers' + ), 'pointer stops at the parent' + + +def test_json_utf8_valid(): + # Multi-byte UTF-8 is not affected: it must round-trip byte for byte. + value = 'caf\u00e9-\U0001f600-\u043d' + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + { + "match": {"headers": {"X-T": value}}, + "action": {"return": 200}, + } + ], + } + ), 'valid utf-8 accepted' + + assert ( + client.conf_get('routes/0/match/headers/X-T') == value + ), 'valid utf-8 round-trips unchanged' + + def test_json_unicode_2(): assert 'success' in client.conf( { From cf071f49a173b726a59da0a0f45e5f2883b43ef6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 07:14:52 +0200 Subject: [PATCH 126/391] utf8: drop the unused case-folding machinery Unit folds case with nxt_strcasecmp() everywhere it actually compares strings. The UTF-8 case-folding path had no production caller at all: nxt_utf8_casecmp() was reachable only from the unit test and the basic fuzzer, nxt_utf8_lowcase() only from nxt_utf8_casecmp() and from utf8_file_name_test, and utf8_file_name_test exists solely to regenerate the tables the other two read. A closed loop of dead code carrying 17160 bytes of tables generated from CaseFolding 6.3.0 (2013), from a generator whose path the comment in nxt_utf8.c got wrong years ago. Keeping it would mean re-deriving the tables against a current Unicode release for a feature nothing uses. nxt_utf8_next() goes with them, unreferenced anywhere in the tree and wrong besides: its loop reads c = *p before testing p < end, so on a buffer whose last byte is a lead byte it dereferences one past the end before deciding to stop. nxt_utf8_decode2() loses its last caller outside this file and becomes static. nxt_utf8_encode(), nxt_utf8_decode(), nxt_utf8_length(), nxt_utf8_sanitize() and nxt_utf8_is_valid() stay; is_valid() has no caller on master today but is the entry point conf validation is about to use for ingress checking. The basic fuzzer loses the casecmp call. nxt_utf8_decode() alone stops after one sequence, so nxt_utf8_length() takes over walking the whole input, which is the coverage casecmp was providing. Measured with gcc, ./configure --openssl: nxt_utf8.o .rodata 12992 -> 0, .data.rel.ro.local 4168 -> 0, .text 1243 -> 987 unitd, stripped 559712 -> 543328 bytes (-16384) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RqWagVeqR5uyNaDfUx6nRR --- auto/make | 12 +- auto/sources | 5 - fuzzing/nxt_basic_fuzz.c | 7 +- src/nxt_unicode_lowcase.h | 1043 ---------------------------- src/nxt_unicode_lowcase.pl | 88 --- src/nxt_unicode_macosx_lowcase.h | 816 ---------------------- src/nxt_utf8.c | 90 +-- src/nxt_utf8.h | 44 -- src/test/nxt_utf8_file_name_test.c | 148 ---- src/test/nxt_utf8_test.c | 11 - 10 files changed, 6 insertions(+), 2258 deletions(-) delete mode 100644 src/nxt_unicode_lowcase.h delete mode 100644 src/nxt_unicode_lowcase.pl delete mode 100644 src/nxt_unicode_macosx_lowcase.h delete mode 100644 src/test/nxt_utf8_file_name_test.c diff --git a/auto/make b/auto/make index e15469bec..509eaaa66 100644 --- a/auto/make +++ b/auto/make @@ -259,7 +259,7 @@ if [ $NXT_TESTS = YES ]; then cat << END >> $NXT_MAKEFILE .PHONY: tests -tests: $NXT_BUILD_DIR/tests $NXT_BUILD_DIR/utf8_file_name_test \\ +tests: $NXT_BUILD_DIR/tests \\ $NXT_BUILD_DIR/ncq_test \\ $NXT_BUILD_DIR/vbcq_test \\ $NXT_BUILD_DIR/unit_app_test $NXT_BUILD_DIR/unit_websocket_chat \\ @@ -276,16 +276,6 @@ $NXT_BUILD_DIR/tests: \$(NXT_TEST_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS -$NXT_BUILD_DIR/utf8_file_name_test: $NXT_LIB_UTF8_FILE_NAME_TEST_SRCS \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ - \$(NXT_OTEL_LIB_STATIC) - \$(PP_LD) \$@ - \$(v)\$(CC) \$(CFLAGS) \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ - -o $NXT_BUILD_DIR/utf8_file_name_test \\ - $NXT_LIB_UTF8_FILE_NAME_TEST_SRCS \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ - $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS - $NXT_BUILD_DIR/ncq_test: $NXT_BUILD_DIR/src/test/nxt_ncq_test.o \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ \$(NXT_OTEL_LIB_STATIC) diff --git a/auto/sources b/auto/sources index 2b78eb4ed..738a5b68c 100644 --- a/auto/sources +++ b/auto/sources @@ -204,11 +204,6 @@ if [ "$NXT_HAVE_CGROUP" = "YES" ]; then fi -NXT_LIB_UTF8_FILE_NAME_TEST_SRCS=" \ - src/test/nxt_utf8_file_name_test.c \ -" - - if [ $NXT_HAVE_ROOTFS = YES ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_fs_mount.c" fi diff --git a/fuzzing/nxt_basic_fuzz.c b/fuzzing/nxt_basic_fuzz.c index 5f71a909c..35e21b6ae 100644 --- a/fuzzing/nxt_basic_fuzz.c +++ b/fuzzing/nxt_basic_fuzz.c @@ -190,10 +190,9 @@ nxt_utf8_fuzz(const u_char *data, size_t size) in = data; nxt_utf8_decode(&in, data + size); - nxt_utf8_casecmp((const u_char *)"ABC АБВ ΑΒΓ", - data, - nxt_length("ABC АБВ ΑΒΓ"), - size); + /* nxt_utf8_decode() stops after one sequence; walk the whole input. */ + + nxt_utf8_length(data, size); } diff --git a/src/nxt_unicode_lowcase.h b/src/nxt_unicode_lowcase.h deleted file mode 100644 index 2f25ab69e..000000000 --- a/src/nxt_unicode_lowcase.h +++ /dev/null @@ -1,1043 +0,0 @@ - -/* - * 26 128-bytes blocks, 521 pointers. - * 14920 bytes on 32-bit platforms, 17000 bytes on 64-bit platforms. - */ - -#define NXT_UNICODE_MAX_LOWCASE 0x10427 - -#define NXT_UNICODE_BLOCK_SIZE 128 - - -static const uint32_t nxt_unicode_block_000[128] nxt_aligned(64) = { - 0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00005, 0x00006, 0x00007, - 0x00008, 0x00009, 0x0000A, 0x0000B, 0x0000C, 0x0000D, 0x0000E, 0x0000F, - 0x00010, 0x00011, 0x00012, 0x00013, 0x00014, 0x00015, 0x00016, 0x00017, - 0x00018, 0x00019, 0x0001A, 0x0001B, 0x0001C, 0x0001D, 0x0001E, 0x0001F, - 0x00020, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, - 0x00028, 0x00029, 0x0002A, 0x0002B, 0x0002C, 0x0002D, 0x0002E, 0x0002F, - 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, - 0x00038, 0x00039, 0x0003A, 0x0003B, 0x0003C, 0x0003D, 0x0003E, 0x0003F, - 0x00040, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, - 0x00068, 0x00069, 0x0006A, 0x0006B, 0x0006C, 0x0006D, 0x0006E, 0x0006F, - 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, - 0x00078, 0x00079, 0x0007A, 0x0005B, 0x0005C, 0x0005D, 0x0005E, 0x0005F, - 0x00060, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, - 0x00068, 0x00069, 0x0006A, 0x0006B, 0x0006C, 0x0006D, 0x0006E, 0x0006F, - 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, - 0x00078, 0x00079, 0x0007A, 0x0007B, 0x0007C, 0x0007D, 0x0007E, 0x0007F, -}; - - -static const uint32_t nxt_unicode_block_001[128] nxt_aligned(64) = { - 0x00080, 0x00081, 0x00082, 0x00083, 0x00084, 0x00085, 0x00086, 0x00087, - 0x00088, 0x00089, 0x0008A, 0x0008B, 0x0008C, 0x0008D, 0x0008E, 0x0008F, - 0x00090, 0x00091, 0x00092, 0x00093, 0x00094, 0x00095, 0x00096, 0x00097, - 0x00098, 0x00099, 0x0009A, 0x0009B, 0x0009C, 0x0009D, 0x0009E, 0x0009F, - 0x000A0, 0x000A1, 0x000A2, 0x000A3, 0x000A4, 0x000A5, 0x000A6, 0x000A7, - 0x000A8, 0x000A9, 0x000AA, 0x000AB, 0x000AC, 0x000AD, 0x000AE, 0x000AF, - 0x000B0, 0x000B1, 0x000B2, 0x000B3, 0x000B4, 0x003BC, 0x000B6, 0x000B7, - 0x000B8, 0x000B9, 0x000BA, 0x000BB, 0x000BC, 0x000BD, 0x000BE, 0x000BF, - 0x000E0, 0x000E1, 0x000E2, 0x000E3, 0x000E4, 0x000E5, 0x000E6, 0x000E7, - 0x000E8, 0x000E9, 0x000EA, 0x000EB, 0x000EC, 0x000ED, 0x000EE, 0x000EF, - 0x000F0, 0x000F1, 0x000F2, 0x000F3, 0x000F4, 0x000F5, 0x000F6, 0x000D7, - 0x000F8, 0x000F9, 0x000FA, 0x000FB, 0x000FC, 0x000FD, 0x000FE, 0x000DF, - 0x000E0, 0x000E1, 0x000E2, 0x000E3, 0x000E4, 0x000E5, 0x000E6, 0x000E7, - 0x000E8, 0x000E9, 0x000EA, 0x000EB, 0x000EC, 0x000ED, 0x000EE, 0x000EF, - 0x000F0, 0x000F1, 0x000F2, 0x000F3, 0x000F4, 0x000F5, 0x000F6, 0x000F7, - 0x000F8, 0x000F9, 0x000FA, 0x000FB, 0x000FC, 0x000FD, 0x000FE, 0x000FF, -}; - - -static const uint32_t nxt_unicode_block_002[128] nxt_aligned(64) = { - 0x00101, 0x00101, 0x00103, 0x00103, 0x00105, 0x00105, 0x00107, 0x00107, - 0x00109, 0x00109, 0x0010B, 0x0010B, 0x0010D, 0x0010D, 0x0010F, 0x0010F, - 0x00111, 0x00111, 0x00113, 0x00113, 0x00115, 0x00115, 0x00117, 0x00117, - 0x00119, 0x00119, 0x0011B, 0x0011B, 0x0011D, 0x0011D, 0x0011F, 0x0011F, - 0x00121, 0x00121, 0x00123, 0x00123, 0x00125, 0x00125, 0x00127, 0x00127, - 0x00129, 0x00129, 0x0012B, 0x0012B, 0x0012D, 0x0012D, 0x0012F, 0x0012F, - 0x00130, 0x00131, 0x00133, 0x00133, 0x00135, 0x00135, 0x00137, 0x00137, - 0x00138, 0x0013A, 0x0013A, 0x0013C, 0x0013C, 0x0013E, 0x0013E, 0x00140, - 0x00140, 0x00142, 0x00142, 0x00144, 0x00144, 0x00146, 0x00146, 0x00148, - 0x00148, 0x00149, 0x0014B, 0x0014B, 0x0014D, 0x0014D, 0x0014F, 0x0014F, - 0x00151, 0x00151, 0x00153, 0x00153, 0x00155, 0x00155, 0x00157, 0x00157, - 0x00159, 0x00159, 0x0015B, 0x0015B, 0x0015D, 0x0015D, 0x0015F, 0x0015F, - 0x00161, 0x00161, 0x00163, 0x00163, 0x00165, 0x00165, 0x00167, 0x00167, - 0x00169, 0x00169, 0x0016B, 0x0016B, 0x0016D, 0x0016D, 0x0016F, 0x0016F, - 0x00171, 0x00171, 0x00173, 0x00173, 0x00175, 0x00175, 0x00177, 0x00177, - 0x000FF, 0x0017A, 0x0017A, 0x0017C, 0x0017C, 0x0017E, 0x0017E, 0x00073, -}; - - -static const uint32_t nxt_unicode_block_003[128] nxt_aligned(64) = { - 0x00180, 0x00253, 0x00183, 0x00183, 0x00185, 0x00185, 0x00254, 0x00188, - 0x00188, 0x00256, 0x00257, 0x0018C, 0x0018C, 0x0018D, 0x001DD, 0x00259, - 0x0025B, 0x00192, 0x00192, 0x00260, 0x00263, 0x00195, 0x00269, 0x00268, - 0x00199, 0x00199, 0x0019A, 0x0019B, 0x0026F, 0x00272, 0x0019E, 0x00275, - 0x001A1, 0x001A1, 0x001A3, 0x001A3, 0x001A5, 0x001A5, 0x00280, 0x001A8, - 0x001A8, 0x00283, 0x001AA, 0x001AB, 0x001AD, 0x001AD, 0x00288, 0x001B0, - 0x001B0, 0x0028A, 0x0028B, 0x001B4, 0x001B4, 0x001B6, 0x001B6, 0x00292, - 0x001B9, 0x001B9, 0x001BA, 0x001BB, 0x001BD, 0x001BD, 0x001BE, 0x001BF, - 0x001C0, 0x001C1, 0x001C2, 0x001C3, 0x001C6, 0x001C6, 0x001C6, 0x001C9, - 0x001C9, 0x001C9, 0x001CC, 0x001CC, 0x001CC, 0x001CE, 0x001CE, 0x001D0, - 0x001D0, 0x001D2, 0x001D2, 0x001D4, 0x001D4, 0x001D6, 0x001D6, 0x001D8, - 0x001D8, 0x001DA, 0x001DA, 0x001DC, 0x001DC, 0x001DD, 0x001DF, 0x001DF, - 0x001E1, 0x001E1, 0x001E3, 0x001E3, 0x001E5, 0x001E5, 0x001E7, 0x001E7, - 0x001E9, 0x001E9, 0x001EB, 0x001EB, 0x001ED, 0x001ED, 0x001EF, 0x001EF, - 0x001F0, 0x001F3, 0x001F3, 0x001F3, 0x001F5, 0x001F5, 0x00195, 0x001BF, - 0x001F9, 0x001F9, 0x001FB, 0x001FB, 0x001FD, 0x001FD, 0x001FF, 0x001FF, -}; - - -static const uint32_t nxt_unicode_block_004[128] nxt_aligned(64) = { - 0x00201, 0x00201, 0x00203, 0x00203, 0x00205, 0x00205, 0x00207, 0x00207, - 0x00209, 0x00209, 0x0020B, 0x0020B, 0x0020D, 0x0020D, 0x0020F, 0x0020F, - 0x00211, 0x00211, 0x00213, 0x00213, 0x00215, 0x00215, 0x00217, 0x00217, - 0x00219, 0x00219, 0x0021B, 0x0021B, 0x0021D, 0x0021D, 0x0021F, 0x0021F, - 0x0019E, 0x00221, 0x00223, 0x00223, 0x00225, 0x00225, 0x00227, 0x00227, - 0x00229, 0x00229, 0x0022B, 0x0022B, 0x0022D, 0x0022D, 0x0022F, 0x0022F, - 0x00231, 0x00231, 0x00233, 0x00233, 0x00234, 0x00235, 0x00236, 0x00237, - 0x00238, 0x00239, 0x02C65, 0x0023C, 0x0023C, 0x0019A, 0x02C66, 0x0023F, - 0x00240, 0x00242, 0x00242, 0x00180, 0x00289, 0x0028C, 0x00247, 0x00247, - 0x00249, 0x00249, 0x0024B, 0x0024B, 0x0024D, 0x0024D, 0x0024F, 0x0024F, - 0x00250, 0x00251, 0x00252, 0x00253, 0x00254, 0x00255, 0x00256, 0x00257, - 0x00258, 0x00259, 0x0025A, 0x0025B, 0x0025C, 0x0025D, 0x0025E, 0x0025F, - 0x00260, 0x00261, 0x00262, 0x00263, 0x00264, 0x00265, 0x00266, 0x00267, - 0x00268, 0x00269, 0x0026A, 0x0026B, 0x0026C, 0x0026D, 0x0026E, 0x0026F, - 0x00270, 0x00271, 0x00272, 0x00273, 0x00274, 0x00275, 0x00276, 0x00277, - 0x00278, 0x00279, 0x0027A, 0x0027B, 0x0027C, 0x0027D, 0x0027E, 0x0027F, -}; - - -static const uint32_t nxt_unicode_block_006[128] nxt_aligned(64) = { - 0x00300, 0x00301, 0x00302, 0x00303, 0x00304, 0x00305, 0x00306, 0x00307, - 0x00308, 0x00309, 0x0030A, 0x0030B, 0x0030C, 0x0030D, 0x0030E, 0x0030F, - 0x00310, 0x00311, 0x00312, 0x00313, 0x00314, 0x00315, 0x00316, 0x00317, - 0x00318, 0x00319, 0x0031A, 0x0031B, 0x0031C, 0x0031D, 0x0031E, 0x0031F, - 0x00320, 0x00321, 0x00322, 0x00323, 0x00324, 0x00325, 0x00326, 0x00327, - 0x00328, 0x00329, 0x0032A, 0x0032B, 0x0032C, 0x0032D, 0x0032E, 0x0032F, - 0x00330, 0x00331, 0x00332, 0x00333, 0x00334, 0x00335, 0x00336, 0x00337, - 0x00338, 0x00339, 0x0033A, 0x0033B, 0x0033C, 0x0033D, 0x0033E, 0x0033F, - 0x00340, 0x00341, 0x00342, 0x00343, 0x00344, 0x003B9, 0x00346, 0x00347, - 0x00348, 0x00349, 0x0034A, 0x0034B, 0x0034C, 0x0034D, 0x0034E, 0x0034F, - 0x00350, 0x00351, 0x00352, 0x00353, 0x00354, 0x00355, 0x00356, 0x00357, - 0x00358, 0x00359, 0x0035A, 0x0035B, 0x0035C, 0x0035D, 0x0035E, 0x0035F, - 0x00360, 0x00361, 0x00362, 0x00363, 0x00364, 0x00365, 0x00366, 0x00367, - 0x00368, 0x00369, 0x0036A, 0x0036B, 0x0036C, 0x0036D, 0x0036E, 0x0036F, - 0x00371, 0x00371, 0x00373, 0x00373, 0x00374, 0x00375, 0x00377, 0x00377, - 0x00378, 0x00379, 0x0037A, 0x0037B, 0x0037C, 0x0037D, 0x0037E, 0x0037F, -}; - - -static const uint32_t nxt_unicode_block_007[128] nxt_aligned(64) = { - 0x00380, 0x00381, 0x00382, 0x00383, 0x00384, 0x00385, 0x003AC, 0x00387, - 0x003AD, 0x003AE, 0x003AF, 0x0038B, 0x003CC, 0x0038D, 0x003CD, 0x003CE, - 0x00390, 0x003B1, 0x003B2, 0x003B3, 0x003B4, 0x003B5, 0x003B6, 0x003B7, - 0x003B8, 0x003B9, 0x003BA, 0x003BB, 0x003BC, 0x003BD, 0x003BE, 0x003BF, - 0x003C0, 0x003C1, 0x003A2, 0x003C3, 0x003C4, 0x003C5, 0x003C6, 0x003C7, - 0x003C8, 0x003C9, 0x003CA, 0x003CB, 0x003AC, 0x003AD, 0x003AE, 0x003AF, - 0x003B0, 0x003B1, 0x003B2, 0x003B3, 0x003B4, 0x003B5, 0x003B6, 0x003B7, - 0x003B8, 0x003B9, 0x003BA, 0x003BB, 0x003BC, 0x003BD, 0x003BE, 0x003BF, - 0x003C0, 0x003C1, 0x003C3, 0x003C3, 0x003C4, 0x003C5, 0x003C6, 0x003C7, - 0x003C8, 0x003C9, 0x003CA, 0x003CB, 0x003CC, 0x003CD, 0x003CE, 0x003D7, - 0x003B2, 0x003B8, 0x003D2, 0x003D3, 0x003D4, 0x003C6, 0x003C0, 0x003D7, - 0x003D9, 0x003D9, 0x003DB, 0x003DB, 0x003DD, 0x003DD, 0x003DF, 0x003DF, - 0x003E1, 0x003E1, 0x003E3, 0x003E3, 0x003E5, 0x003E5, 0x003E7, 0x003E7, - 0x003E9, 0x003E9, 0x003EB, 0x003EB, 0x003ED, 0x003ED, 0x003EF, 0x003EF, - 0x003BA, 0x003C1, 0x003F2, 0x003F3, 0x003B8, 0x003B5, 0x003F6, 0x003F8, - 0x003F8, 0x003F2, 0x003FB, 0x003FB, 0x003FC, 0x0037B, 0x0037C, 0x0037D, -}; - - -static const uint32_t nxt_unicode_block_008[128] nxt_aligned(64) = { - 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, - 0x00458, 0x00459, 0x0045A, 0x0045B, 0x0045C, 0x0045D, 0x0045E, 0x0045F, - 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, - 0x00438, 0x00439, 0x0043A, 0x0043B, 0x0043C, 0x0043D, 0x0043E, 0x0043F, - 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, - 0x00448, 0x00449, 0x0044A, 0x0044B, 0x0044C, 0x0044D, 0x0044E, 0x0044F, - 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, - 0x00438, 0x00439, 0x0043A, 0x0043B, 0x0043C, 0x0043D, 0x0043E, 0x0043F, - 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, - 0x00448, 0x00449, 0x0044A, 0x0044B, 0x0044C, 0x0044D, 0x0044E, 0x0044F, - 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, - 0x00458, 0x00459, 0x0045A, 0x0045B, 0x0045C, 0x0045D, 0x0045E, 0x0045F, - 0x00461, 0x00461, 0x00463, 0x00463, 0x00465, 0x00465, 0x00467, 0x00467, - 0x00469, 0x00469, 0x0046B, 0x0046B, 0x0046D, 0x0046D, 0x0046F, 0x0046F, - 0x00471, 0x00471, 0x00473, 0x00473, 0x00475, 0x00475, 0x00477, 0x00477, - 0x00479, 0x00479, 0x0047B, 0x0047B, 0x0047D, 0x0047D, 0x0047F, 0x0047F, -}; - - -static const uint32_t nxt_unicode_block_009[128] nxt_aligned(64) = { - 0x00481, 0x00481, 0x00482, 0x00483, 0x00484, 0x00485, 0x00486, 0x00487, - 0x00488, 0x00489, 0x0048B, 0x0048B, 0x0048D, 0x0048D, 0x0048F, 0x0048F, - 0x00491, 0x00491, 0x00493, 0x00493, 0x00495, 0x00495, 0x00497, 0x00497, - 0x00499, 0x00499, 0x0049B, 0x0049B, 0x0049D, 0x0049D, 0x0049F, 0x0049F, - 0x004A1, 0x004A1, 0x004A3, 0x004A3, 0x004A5, 0x004A5, 0x004A7, 0x004A7, - 0x004A9, 0x004A9, 0x004AB, 0x004AB, 0x004AD, 0x004AD, 0x004AF, 0x004AF, - 0x004B1, 0x004B1, 0x004B3, 0x004B3, 0x004B5, 0x004B5, 0x004B7, 0x004B7, - 0x004B9, 0x004B9, 0x004BB, 0x004BB, 0x004BD, 0x004BD, 0x004BF, 0x004BF, - 0x004CF, 0x004C2, 0x004C2, 0x004C4, 0x004C4, 0x004C6, 0x004C6, 0x004C8, - 0x004C8, 0x004CA, 0x004CA, 0x004CC, 0x004CC, 0x004CE, 0x004CE, 0x004CF, - 0x004D1, 0x004D1, 0x004D3, 0x004D3, 0x004D5, 0x004D5, 0x004D7, 0x004D7, - 0x004D9, 0x004D9, 0x004DB, 0x004DB, 0x004DD, 0x004DD, 0x004DF, 0x004DF, - 0x004E1, 0x004E1, 0x004E3, 0x004E3, 0x004E5, 0x004E5, 0x004E7, 0x004E7, - 0x004E9, 0x004E9, 0x004EB, 0x004EB, 0x004ED, 0x004ED, 0x004EF, 0x004EF, - 0x004F1, 0x004F1, 0x004F3, 0x004F3, 0x004F5, 0x004F5, 0x004F7, 0x004F7, - 0x004F9, 0x004F9, 0x004FB, 0x004FB, 0x004FD, 0x004FD, 0x004FF, 0x004FF, -}; - - -static const uint32_t nxt_unicode_block_00a[128] nxt_aligned(64) = { - 0x00501, 0x00501, 0x00503, 0x00503, 0x00505, 0x00505, 0x00507, 0x00507, - 0x00509, 0x00509, 0x0050B, 0x0050B, 0x0050D, 0x0050D, 0x0050F, 0x0050F, - 0x00511, 0x00511, 0x00513, 0x00513, 0x00515, 0x00515, 0x00517, 0x00517, - 0x00519, 0x00519, 0x0051B, 0x0051B, 0x0051D, 0x0051D, 0x0051F, 0x0051F, - 0x00521, 0x00521, 0x00523, 0x00523, 0x00525, 0x00525, 0x00527, 0x00527, - 0x00528, 0x00529, 0x0052A, 0x0052B, 0x0052C, 0x0052D, 0x0052E, 0x0052F, - 0x00530, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, - 0x00568, 0x00569, 0x0056A, 0x0056B, 0x0056C, 0x0056D, 0x0056E, 0x0056F, - 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, - 0x00578, 0x00579, 0x0057A, 0x0057B, 0x0057C, 0x0057D, 0x0057E, 0x0057F, - 0x00580, 0x00581, 0x00582, 0x00583, 0x00584, 0x00585, 0x00586, 0x00557, - 0x00558, 0x00559, 0x0055A, 0x0055B, 0x0055C, 0x0055D, 0x0055E, 0x0055F, - 0x00560, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, - 0x00568, 0x00569, 0x0056A, 0x0056B, 0x0056C, 0x0056D, 0x0056E, 0x0056F, - 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, - 0x00578, 0x00579, 0x0057A, 0x0057B, 0x0057C, 0x0057D, 0x0057E, 0x0057F, -}; - - -static const uint32_t nxt_unicode_block_021[128] nxt_aligned(64) = { - 0x01080, 0x01081, 0x01082, 0x01083, 0x01084, 0x01085, 0x01086, 0x01087, - 0x01088, 0x01089, 0x0108A, 0x0108B, 0x0108C, 0x0108D, 0x0108E, 0x0108F, - 0x01090, 0x01091, 0x01092, 0x01093, 0x01094, 0x01095, 0x01096, 0x01097, - 0x01098, 0x01099, 0x0109A, 0x0109B, 0x0109C, 0x0109D, 0x0109E, 0x0109F, - 0x02D00, 0x02D01, 0x02D02, 0x02D03, 0x02D04, 0x02D05, 0x02D06, 0x02D07, - 0x02D08, 0x02D09, 0x02D0A, 0x02D0B, 0x02D0C, 0x02D0D, 0x02D0E, 0x02D0F, - 0x02D10, 0x02D11, 0x02D12, 0x02D13, 0x02D14, 0x02D15, 0x02D16, 0x02D17, - 0x02D18, 0x02D19, 0x02D1A, 0x02D1B, 0x02D1C, 0x02D1D, 0x02D1E, 0x02D1F, - 0x02D20, 0x02D21, 0x02D22, 0x02D23, 0x02D24, 0x02D25, 0x010C6, 0x02D27, - 0x010C8, 0x010C9, 0x010CA, 0x010CB, 0x010CC, 0x02D2D, 0x010CE, 0x010CF, - 0x010D0, 0x010D1, 0x010D2, 0x010D3, 0x010D4, 0x010D5, 0x010D6, 0x010D7, - 0x010D8, 0x010D9, 0x010DA, 0x010DB, 0x010DC, 0x010DD, 0x010DE, 0x010DF, - 0x010E0, 0x010E1, 0x010E2, 0x010E3, 0x010E4, 0x010E5, 0x010E6, 0x010E7, - 0x010E8, 0x010E9, 0x010EA, 0x010EB, 0x010EC, 0x010ED, 0x010EE, 0x010EF, - 0x010F0, 0x010F1, 0x010F2, 0x010F3, 0x010F4, 0x010F5, 0x010F6, 0x010F7, - 0x010F8, 0x010F9, 0x010FA, 0x010FB, 0x010FC, 0x010FD, 0x010FE, 0x010FF, -}; - - -static const uint32_t nxt_unicode_block_03c[128] nxt_aligned(64) = { - 0x01E01, 0x01E01, 0x01E03, 0x01E03, 0x01E05, 0x01E05, 0x01E07, 0x01E07, - 0x01E09, 0x01E09, 0x01E0B, 0x01E0B, 0x01E0D, 0x01E0D, 0x01E0F, 0x01E0F, - 0x01E11, 0x01E11, 0x01E13, 0x01E13, 0x01E15, 0x01E15, 0x01E17, 0x01E17, - 0x01E19, 0x01E19, 0x01E1B, 0x01E1B, 0x01E1D, 0x01E1D, 0x01E1F, 0x01E1F, - 0x01E21, 0x01E21, 0x01E23, 0x01E23, 0x01E25, 0x01E25, 0x01E27, 0x01E27, - 0x01E29, 0x01E29, 0x01E2B, 0x01E2B, 0x01E2D, 0x01E2D, 0x01E2F, 0x01E2F, - 0x01E31, 0x01E31, 0x01E33, 0x01E33, 0x01E35, 0x01E35, 0x01E37, 0x01E37, - 0x01E39, 0x01E39, 0x01E3B, 0x01E3B, 0x01E3D, 0x01E3D, 0x01E3F, 0x01E3F, - 0x01E41, 0x01E41, 0x01E43, 0x01E43, 0x01E45, 0x01E45, 0x01E47, 0x01E47, - 0x01E49, 0x01E49, 0x01E4B, 0x01E4B, 0x01E4D, 0x01E4D, 0x01E4F, 0x01E4F, - 0x01E51, 0x01E51, 0x01E53, 0x01E53, 0x01E55, 0x01E55, 0x01E57, 0x01E57, - 0x01E59, 0x01E59, 0x01E5B, 0x01E5B, 0x01E5D, 0x01E5D, 0x01E5F, 0x01E5F, - 0x01E61, 0x01E61, 0x01E63, 0x01E63, 0x01E65, 0x01E65, 0x01E67, 0x01E67, - 0x01E69, 0x01E69, 0x01E6B, 0x01E6B, 0x01E6D, 0x01E6D, 0x01E6F, 0x01E6F, - 0x01E71, 0x01E71, 0x01E73, 0x01E73, 0x01E75, 0x01E75, 0x01E77, 0x01E77, - 0x01E79, 0x01E79, 0x01E7B, 0x01E7B, 0x01E7D, 0x01E7D, 0x01E7F, 0x01E7F, -}; - - -static const uint32_t nxt_unicode_block_03d[128] nxt_aligned(64) = { - 0x01E81, 0x01E81, 0x01E83, 0x01E83, 0x01E85, 0x01E85, 0x01E87, 0x01E87, - 0x01E89, 0x01E89, 0x01E8B, 0x01E8B, 0x01E8D, 0x01E8D, 0x01E8F, 0x01E8F, - 0x01E91, 0x01E91, 0x01E93, 0x01E93, 0x01E95, 0x01E95, 0x01E96, 0x01E97, - 0x01E98, 0x01E99, 0x01E9A, 0x01E61, 0x01E9C, 0x01E9D, 0x000DF, 0x01E9F, - 0x01EA1, 0x01EA1, 0x01EA3, 0x01EA3, 0x01EA5, 0x01EA5, 0x01EA7, 0x01EA7, - 0x01EA9, 0x01EA9, 0x01EAB, 0x01EAB, 0x01EAD, 0x01EAD, 0x01EAF, 0x01EAF, - 0x01EB1, 0x01EB1, 0x01EB3, 0x01EB3, 0x01EB5, 0x01EB5, 0x01EB7, 0x01EB7, - 0x01EB9, 0x01EB9, 0x01EBB, 0x01EBB, 0x01EBD, 0x01EBD, 0x01EBF, 0x01EBF, - 0x01EC1, 0x01EC1, 0x01EC3, 0x01EC3, 0x01EC5, 0x01EC5, 0x01EC7, 0x01EC7, - 0x01EC9, 0x01EC9, 0x01ECB, 0x01ECB, 0x01ECD, 0x01ECD, 0x01ECF, 0x01ECF, - 0x01ED1, 0x01ED1, 0x01ED3, 0x01ED3, 0x01ED5, 0x01ED5, 0x01ED7, 0x01ED7, - 0x01ED9, 0x01ED9, 0x01EDB, 0x01EDB, 0x01EDD, 0x01EDD, 0x01EDF, 0x01EDF, - 0x01EE1, 0x01EE1, 0x01EE3, 0x01EE3, 0x01EE5, 0x01EE5, 0x01EE7, 0x01EE7, - 0x01EE9, 0x01EE9, 0x01EEB, 0x01EEB, 0x01EED, 0x01EED, 0x01EEF, 0x01EEF, - 0x01EF1, 0x01EF1, 0x01EF3, 0x01EF3, 0x01EF5, 0x01EF5, 0x01EF7, 0x01EF7, - 0x01EF9, 0x01EF9, 0x01EFB, 0x01EFB, 0x01EFD, 0x01EFD, 0x01EFF, 0x01EFF, -}; - - -static const uint32_t nxt_unicode_block_03e[128] nxt_aligned(64) = { - 0x01F00, 0x01F01, 0x01F02, 0x01F03, 0x01F04, 0x01F05, 0x01F06, 0x01F07, - 0x01F00, 0x01F01, 0x01F02, 0x01F03, 0x01F04, 0x01F05, 0x01F06, 0x01F07, - 0x01F10, 0x01F11, 0x01F12, 0x01F13, 0x01F14, 0x01F15, 0x01F16, 0x01F17, - 0x01F10, 0x01F11, 0x01F12, 0x01F13, 0x01F14, 0x01F15, 0x01F1E, 0x01F1F, - 0x01F20, 0x01F21, 0x01F22, 0x01F23, 0x01F24, 0x01F25, 0x01F26, 0x01F27, - 0x01F20, 0x01F21, 0x01F22, 0x01F23, 0x01F24, 0x01F25, 0x01F26, 0x01F27, - 0x01F30, 0x01F31, 0x01F32, 0x01F33, 0x01F34, 0x01F35, 0x01F36, 0x01F37, - 0x01F30, 0x01F31, 0x01F32, 0x01F33, 0x01F34, 0x01F35, 0x01F36, 0x01F37, - 0x01F40, 0x01F41, 0x01F42, 0x01F43, 0x01F44, 0x01F45, 0x01F46, 0x01F47, - 0x01F40, 0x01F41, 0x01F42, 0x01F43, 0x01F44, 0x01F45, 0x01F4E, 0x01F4F, - 0x01F50, 0x01F51, 0x01F52, 0x01F53, 0x01F54, 0x01F55, 0x01F56, 0x01F57, - 0x01F58, 0x01F51, 0x01F5A, 0x01F53, 0x01F5C, 0x01F55, 0x01F5E, 0x01F57, - 0x01F60, 0x01F61, 0x01F62, 0x01F63, 0x01F64, 0x01F65, 0x01F66, 0x01F67, - 0x01F60, 0x01F61, 0x01F62, 0x01F63, 0x01F64, 0x01F65, 0x01F66, 0x01F67, - 0x01F70, 0x01F71, 0x01F72, 0x01F73, 0x01F74, 0x01F75, 0x01F76, 0x01F77, - 0x01F78, 0x01F79, 0x01F7A, 0x01F7B, 0x01F7C, 0x01F7D, 0x01F7E, 0x01F7F, -}; - - -static const uint32_t nxt_unicode_block_03f[128] nxt_aligned(64) = { - 0x01F80, 0x01F81, 0x01F82, 0x01F83, 0x01F84, 0x01F85, 0x01F86, 0x01F87, - 0x01F80, 0x01F81, 0x01F82, 0x01F83, 0x01F84, 0x01F85, 0x01F86, 0x01F87, - 0x01F90, 0x01F91, 0x01F92, 0x01F93, 0x01F94, 0x01F95, 0x01F96, 0x01F97, - 0x01F90, 0x01F91, 0x01F92, 0x01F93, 0x01F94, 0x01F95, 0x01F96, 0x01F97, - 0x01FA0, 0x01FA1, 0x01FA2, 0x01FA3, 0x01FA4, 0x01FA5, 0x01FA6, 0x01FA7, - 0x01FA0, 0x01FA1, 0x01FA2, 0x01FA3, 0x01FA4, 0x01FA5, 0x01FA6, 0x01FA7, - 0x01FB0, 0x01FB1, 0x01FB2, 0x01FB3, 0x01FB4, 0x01FB5, 0x01FB6, 0x01FB7, - 0x01FB0, 0x01FB1, 0x01F70, 0x01F71, 0x01FB3, 0x01FBD, 0x003B9, 0x01FBF, - 0x01FC0, 0x01FC1, 0x01FC2, 0x01FC3, 0x01FC4, 0x01FC5, 0x01FC6, 0x01FC7, - 0x01F72, 0x01F73, 0x01F74, 0x01F75, 0x01FC3, 0x01FCD, 0x01FCE, 0x01FCF, - 0x01FD0, 0x01FD1, 0x01FD2, 0x01FD3, 0x01FD4, 0x01FD5, 0x01FD6, 0x01FD7, - 0x01FD0, 0x01FD1, 0x01F76, 0x01F77, 0x01FDC, 0x01FDD, 0x01FDE, 0x01FDF, - 0x01FE0, 0x01FE1, 0x01FE2, 0x01FE3, 0x01FE4, 0x01FE5, 0x01FE6, 0x01FE7, - 0x01FE0, 0x01FE1, 0x01F7A, 0x01F7B, 0x01FE5, 0x01FED, 0x01FEE, 0x01FEF, - 0x01FF0, 0x01FF1, 0x01FF2, 0x01FF3, 0x01FF4, 0x01FF5, 0x01FF6, 0x01FF7, - 0x01F78, 0x01F79, 0x01F7C, 0x01F7D, 0x01FF3, 0x01FFD, 0x01FFE, 0x01FFF, -}; - - -static const uint32_t nxt_unicode_block_042[128] nxt_aligned(64) = { - 0x02100, 0x02101, 0x02102, 0x02103, 0x02104, 0x02105, 0x02106, 0x02107, - 0x02108, 0x02109, 0x0210A, 0x0210B, 0x0210C, 0x0210D, 0x0210E, 0x0210F, - 0x02110, 0x02111, 0x02112, 0x02113, 0x02114, 0x02115, 0x02116, 0x02117, - 0x02118, 0x02119, 0x0211A, 0x0211B, 0x0211C, 0x0211D, 0x0211E, 0x0211F, - 0x02120, 0x02121, 0x02122, 0x02123, 0x02124, 0x02125, 0x003C9, 0x02127, - 0x02128, 0x02129, 0x0006B, 0x000E5, 0x0212C, 0x0212D, 0x0212E, 0x0212F, - 0x02130, 0x02131, 0x0214E, 0x02133, 0x02134, 0x02135, 0x02136, 0x02137, - 0x02138, 0x02139, 0x0213A, 0x0213B, 0x0213C, 0x0213D, 0x0213E, 0x0213F, - 0x02140, 0x02141, 0x02142, 0x02143, 0x02144, 0x02145, 0x02146, 0x02147, - 0x02148, 0x02149, 0x0214A, 0x0214B, 0x0214C, 0x0214D, 0x0214E, 0x0214F, - 0x02150, 0x02151, 0x02152, 0x02153, 0x02154, 0x02155, 0x02156, 0x02157, - 0x02158, 0x02159, 0x0215A, 0x0215B, 0x0215C, 0x0215D, 0x0215E, 0x0215F, - 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, - 0x02178, 0x02179, 0x0217A, 0x0217B, 0x0217C, 0x0217D, 0x0217E, 0x0217F, - 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, - 0x02178, 0x02179, 0x0217A, 0x0217B, 0x0217C, 0x0217D, 0x0217E, 0x0217F, -}; - - -static const uint32_t nxt_unicode_block_043[128] nxt_aligned(64) = { - 0x02180, 0x02181, 0x02182, 0x02184, 0x02184, 0x02185, 0x02186, 0x02187, - 0x02188, 0x02189, 0x0218A, 0x0218B, 0x0218C, 0x0218D, 0x0218E, 0x0218F, - 0x02190, 0x02191, 0x02192, 0x02193, 0x02194, 0x02195, 0x02196, 0x02197, - 0x02198, 0x02199, 0x0219A, 0x0219B, 0x0219C, 0x0219D, 0x0219E, 0x0219F, - 0x021A0, 0x021A1, 0x021A2, 0x021A3, 0x021A4, 0x021A5, 0x021A6, 0x021A7, - 0x021A8, 0x021A9, 0x021AA, 0x021AB, 0x021AC, 0x021AD, 0x021AE, 0x021AF, - 0x021B0, 0x021B1, 0x021B2, 0x021B3, 0x021B4, 0x021B5, 0x021B6, 0x021B7, - 0x021B8, 0x021B9, 0x021BA, 0x021BB, 0x021BC, 0x021BD, 0x021BE, 0x021BF, - 0x021C0, 0x021C1, 0x021C2, 0x021C3, 0x021C4, 0x021C5, 0x021C6, 0x021C7, - 0x021C8, 0x021C9, 0x021CA, 0x021CB, 0x021CC, 0x021CD, 0x021CE, 0x021CF, - 0x021D0, 0x021D1, 0x021D2, 0x021D3, 0x021D4, 0x021D5, 0x021D6, 0x021D7, - 0x021D8, 0x021D9, 0x021DA, 0x021DB, 0x021DC, 0x021DD, 0x021DE, 0x021DF, - 0x021E0, 0x021E1, 0x021E2, 0x021E3, 0x021E4, 0x021E5, 0x021E6, 0x021E7, - 0x021E8, 0x021E9, 0x021EA, 0x021EB, 0x021EC, 0x021ED, 0x021EE, 0x021EF, - 0x021F0, 0x021F1, 0x021F2, 0x021F3, 0x021F4, 0x021F5, 0x021F6, 0x021F7, - 0x021F8, 0x021F9, 0x021FA, 0x021FB, 0x021FC, 0x021FD, 0x021FE, 0x021FF, -}; - - -static const uint32_t nxt_unicode_block_049[128] nxt_aligned(64) = { - 0x02480, 0x02481, 0x02482, 0x02483, 0x02484, 0x02485, 0x02486, 0x02487, - 0x02488, 0x02489, 0x0248A, 0x0248B, 0x0248C, 0x0248D, 0x0248E, 0x0248F, - 0x02490, 0x02491, 0x02492, 0x02493, 0x02494, 0x02495, 0x02496, 0x02497, - 0x02498, 0x02499, 0x0249A, 0x0249B, 0x0249C, 0x0249D, 0x0249E, 0x0249F, - 0x024A0, 0x024A1, 0x024A2, 0x024A3, 0x024A4, 0x024A5, 0x024A6, 0x024A7, - 0x024A8, 0x024A9, 0x024AA, 0x024AB, 0x024AC, 0x024AD, 0x024AE, 0x024AF, - 0x024B0, 0x024B1, 0x024B2, 0x024B3, 0x024B4, 0x024B5, 0x024D0, 0x024D1, - 0x024D2, 0x024D3, 0x024D4, 0x024D5, 0x024D6, 0x024D7, 0x024D8, 0x024D9, - 0x024DA, 0x024DB, 0x024DC, 0x024DD, 0x024DE, 0x024DF, 0x024E0, 0x024E1, - 0x024E2, 0x024E3, 0x024E4, 0x024E5, 0x024E6, 0x024E7, 0x024E8, 0x024E9, - 0x024D0, 0x024D1, 0x024D2, 0x024D3, 0x024D4, 0x024D5, 0x024D6, 0x024D7, - 0x024D8, 0x024D9, 0x024DA, 0x024DB, 0x024DC, 0x024DD, 0x024DE, 0x024DF, - 0x024E0, 0x024E1, 0x024E2, 0x024E3, 0x024E4, 0x024E5, 0x024E6, 0x024E7, - 0x024E8, 0x024E9, 0x024EA, 0x024EB, 0x024EC, 0x024ED, 0x024EE, 0x024EF, - 0x024F0, 0x024F1, 0x024F2, 0x024F3, 0x024F4, 0x024F5, 0x024F6, 0x024F7, - 0x024F8, 0x024F9, 0x024FA, 0x024FB, 0x024FC, 0x024FD, 0x024FE, 0x024FF, -}; - - -static const uint32_t nxt_unicode_block_058[128] nxt_aligned(64) = { - 0x02C30, 0x02C31, 0x02C32, 0x02C33, 0x02C34, 0x02C35, 0x02C36, 0x02C37, - 0x02C38, 0x02C39, 0x02C3A, 0x02C3B, 0x02C3C, 0x02C3D, 0x02C3E, 0x02C3F, - 0x02C40, 0x02C41, 0x02C42, 0x02C43, 0x02C44, 0x02C45, 0x02C46, 0x02C47, - 0x02C48, 0x02C49, 0x02C4A, 0x02C4B, 0x02C4C, 0x02C4D, 0x02C4E, 0x02C4F, - 0x02C50, 0x02C51, 0x02C52, 0x02C53, 0x02C54, 0x02C55, 0x02C56, 0x02C57, - 0x02C58, 0x02C59, 0x02C5A, 0x02C5B, 0x02C5C, 0x02C5D, 0x02C5E, 0x02C2F, - 0x02C30, 0x02C31, 0x02C32, 0x02C33, 0x02C34, 0x02C35, 0x02C36, 0x02C37, - 0x02C38, 0x02C39, 0x02C3A, 0x02C3B, 0x02C3C, 0x02C3D, 0x02C3E, 0x02C3F, - 0x02C40, 0x02C41, 0x02C42, 0x02C43, 0x02C44, 0x02C45, 0x02C46, 0x02C47, - 0x02C48, 0x02C49, 0x02C4A, 0x02C4B, 0x02C4C, 0x02C4D, 0x02C4E, 0x02C4F, - 0x02C50, 0x02C51, 0x02C52, 0x02C53, 0x02C54, 0x02C55, 0x02C56, 0x02C57, - 0x02C58, 0x02C59, 0x02C5A, 0x02C5B, 0x02C5C, 0x02C5D, 0x02C5E, 0x02C5F, - 0x02C61, 0x02C61, 0x0026B, 0x01D7D, 0x0027D, 0x02C65, 0x02C66, 0x02C68, - 0x02C68, 0x02C6A, 0x02C6A, 0x02C6C, 0x02C6C, 0x00251, 0x00271, 0x00250, - 0x00252, 0x02C71, 0x02C73, 0x02C73, 0x02C74, 0x02C76, 0x02C76, 0x02C77, - 0x02C78, 0x02C79, 0x02C7A, 0x02C7B, 0x02C7C, 0x02C7D, 0x0023F, 0x00240, -}; - - -static const uint32_t nxt_unicode_block_059[128] nxt_aligned(64) = { - 0x02C81, 0x02C81, 0x02C83, 0x02C83, 0x02C85, 0x02C85, 0x02C87, 0x02C87, - 0x02C89, 0x02C89, 0x02C8B, 0x02C8B, 0x02C8D, 0x02C8D, 0x02C8F, 0x02C8F, - 0x02C91, 0x02C91, 0x02C93, 0x02C93, 0x02C95, 0x02C95, 0x02C97, 0x02C97, - 0x02C99, 0x02C99, 0x02C9B, 0x02C9B, 0x02C9D, 0x02C9D, 0x02C9F, 0x02C9F, - 0x02CA1, 0x02CA1, 0x02CA3, 0x02CA3, 0x02CA5, 0x02CA5, 0x02CA7, 0x02CA7, - 0x02CA9, 0x02CA9, 0x02CAB, 0x02CAB, 0x02CAD, 0x02CAD, 0x02CAF, 0x02CAF, - 0x02CB1, 0x02CB1, 0x02CB3, 0x02CB3, 0x02CB5, 0x02CB5, 0x02CB7, 0x02CB7, - 0x02CB9, 0x02CB9, 0x02CBB, 0x02CBB, 0x02CBD, 0x02CBD, 0x02CBF, 0x02CBF, - 0x02CC1, 0x02CC1, 0x02CC3, 0x02CC3, 0x02CC5, 0x02CC5, 0x02CC7, 0x02CC7, - 0x02CC9, 0x02CC9, 0x02CCB, 0x02CCB, 0x02CCD, 0x02CCD, 0x02CCF, 0x02CCF, - 0x02CD1, 0x02CD1, 0x02CD3, 0x02CD3, 0x02CD5, 0x02CD5, 0x02CD7, 0x02CD7, - 0x02CD9, 0x02CD9, 0x02CDB, 0x02CDB, 0x02CDD, 0x02CDD, 0x02CDF, 0x02CDF, - 0x02CE1, 0x02CE1, 0x02CE3, 0x02CE3, 0x02CE4, 0x02CE5, 0x02CE6, 0x02CE7, - 0x02CE8, 0x02CE9, 0x02CEA, 0x02CEC, 0x02CEC, 0x02CEE, 0x02CEE, 0x02CEF, - 0x02CF0, 0x02CF1, 0x02CF3, 0x02CF3, 0x02CF4, 0x02CF5, 0x02CF6, 0x02CF7, - 0x02CF8, 0x02CF9, 0x02CFA, 0x02CFB, 0x02CFC, 0x02CFD, 0x02CFE, 0x02CFF, -}; - - -static const uint32_t nxt_unicode_block_14c[128] nxt_aligned(64) = { - 0x0A600, 0x0A601, 0x0A602, 0x0A603, 0x0A604, 0x0A605, 0x0A606, 0x0A607, - 0x0A608, 0x0A609, 0x0A60A, 0x0A60B, 0x0A60C, 0x0A60D, 0x0A60E, 0x0A60F, - 0x0A610, 0x0A611, 0x0A612, 0x0A613, 0x0A614, 0x0A615, 0x0A616, 0x0A617, - 0x0A618, 0x0A619, 0x0A61A, 0x0A61B, 0x0A61C, 0x0A61D, 0x0A61E, 0x0A61F, - 0x0A620, 0x0A621, 0x0A622, 0x0A623, 0x0A624, 0x0A625, 0x0A626, 0x0A627, - 0x0A628, 0x0A629, 0x0A62A, 0x0A62B, 0x0A62C, 0x0A62D, 0x0A62E, 0x0A62F, - 0x0A630, 0x0A631, 0x0A632, 0x0A633, 0x0A634, 0x0A635, 0x0A636, 0x0A637, - 0x0A638, 0x0A639, 0x0A63A, 0x0A63B, 0x0A63C, 0x0A63D, 0x0A63E, 0x0A63F, - 0x0A641, 0x0A641, 0x0A643, 0x0A643, 0x0A645, 0x0A645, 0x0A647, 0x0A647, - 0x0A649, 0x0A649, 0x0A64B, 0x0A64B, 0x0A64D, 0x0A64D, 0x0A64F, 0x0A64F, - 0x0A651, 0x0A651, 0x0A653, 0x0A653, 0x0A655, 0x0A655, 0x0A657, 0x0A657, - 0x0A659, 0x0A659, 0x0A65B, 0x0A65B, 0x0A65D, 0x0A65D, 0x0A65F, 0x0A65F, - 0x0A661, 0x0A661, 0x0A663, 0x0A663, 0x0A665, 0x0A665, 0x0A667, 0x0A667, - 0x0A669, 0x0A669, 0x0A66B, 0x0A66B, 0x0A66D, 0x0A66D, 0x0A66E, 0x0A66F, - 0x0A670, 0x0A671, 0x0A672, 0x0A673, 0x0A674, 0x0A675, 0x0A676, 0x0A677, - 0x0A678, 0x0A679, 0x0A67A, 0x0A67B, 0x0A67C, 0x0A67D, 0x0A67E, 0x0A67F, -}; - - -static const uint32_t nxt_unicode_block_14d[128] nxt_aligned(64) = { - 0x0A681, 0x0A681, 0x0A683, 0x0A683, 0x0A685, 0x0A685, 0x0A687, 0x0A687, - 0x0A689, 0x0A689, 0x0A68B, 0x0A68B, 0x0A68D, 0x0A68D, 0x0A68F, 0x0A68F, - 0x0A691, 0x0A691, 0x0A693, 0x0A693, 0x0A695, 0x0A695, 0x0A697, 0x0A697, - 0x0A698, 0x0A699, 0x0A69A, 0x0A69B, 0x0A69C, 0x0A69D, 0x0A69E, 0x0A69F, - 0x0A6A0, 0x0A6A1, 0x0A6A2, 0x0A6A3, 0x0A6A4, 0x0A6A5, 0x0A6A6, 0x0A6A7, - 0x0A6A8, 0x0A6A9, 0x0A6AA, 0x0A6AB, 0x0A6AC, 0x0A6AD, 0x0A6AE, 0x0A6AF, - 0x0A6B0, 0x0A6B1, 0x0A6B2, 0x0A6B3, 0x0A6B4, 0x0A6B5, 0x0A6B6, 0x0A6B7, - 0x0A6B8, 0x0A6B9, 0x0A6BA, 0x0A6BB, 0x0A6BC, 0x0A6BD, 0x0A6BE, 0x0A6BF, - 0x0A6C0, 0x0A6C1, 0x0A6C2, 0x0A6C3, 0x0A6C4, 0x0A6C5, 0x0A6C6, 0x0A6C7, - 0x0A6C8, 0x0A6C9, 0x0A6CA, 0x0A6CB, 0x0A6CC, 0x0A6CD, 0x0A6CE, 0x0A6CF, - 0x0A6D0, 0x0A6D1, 0x0A6D2, 0x0A6D3, 0x0A6D4, 0x0A6D5, 0x0A6D6, 0x0A6D7, - 0x0A6D8, 0x0A6D9, 0x0A6DA, 0x0A6DB, 0x0A6DC, 0x0A6DD, 0x0A6DE, 0x0A6DF, - 0x0A6E0, 0x0A6E1, 0x0A6E2, 0x0A6E3, 0x0A6E4, 0x0A6E5, 0x0A6E6, 0x0A6E7, - 0x0A6E8, 0x0A6E9, 0x0A6EA, 0x0A6EB, 0x0A6EC, 0x0A6ED, 0x0A6EE, 0x0A6EF, - 0x0A6F0, 0x0A6F1, 0x0A6F2, 0x0A6F3, 0x0A6F4, 0x0A6F5, 0x0A6F6, 0x0A6F7, - 0x0A6F8, 0x0A6F9, 0x0A6FA, 0x0A6FB, 0x0A6FC, 0x0A6FD, 0x0A6FE, 0x0A6FF, -}; - - -static const uint32_t nxt_unicode_block_14e[128] nxt_aligned(64) = { - 0x0A700, 0x0A701, 0x0A702, 0x0A703, 0x0A704, 0x0A705, 0x0A706, 0x0A707, - 0x0A708, 0x0A709, 0x0A70A, 0x0A70B, 0x0A70C, 0x0A70D, 0x0A70E, 0x0A70F, - 0x0A710, 0x0A711, 0x0A712, 0x0A713, 0x0A714, 0x0A715, 0x0A716, 0x0A717, - 0x0A718, 0x0A719, 0x0A71A, 0x0A71B, 0x0A71C, 0x0A71D, 0x0A71E, 0x0A71F, - 0x0A720, 0x0A721, 0x0A723, 0x0A723, 0x0A725, 0x0A725, 0x0A727, 0x0A727, - 0x0A729, 0x0A729, 0x0A72B, 0x0A72B, 0x0A72D, 0x0A72D, 0x0A72F, 0x0A72F, - 0x0A730, 0x0A731, 0x0A733, 0x0A733, 0x0A735, 0x0A735, 0x0A737, 0x0A737, - 0x0A739, 0x0A739, 0x0A73B, 0x0A73B, 0x0A73D, 0x0A73D, 0x0A73F, 0x0A73F, - 0x0A741, 0x0A741, 0x0A743, 0x0A743, 0x0A745, 0x0A745, 0x0A747, 0x0A747, - 0x0A749, 0x0A749, 0x0A74B, 0x0A74B, 0x0A74D, 0x0A74D, 0x0A74F, 0x0A74F, - 0x0A751, 0x0A751, 0x0A753, 0x0A753, 0x0A755, 0x0A755, 0x0A757, 0x0A757, - 0x0A759, 0x0A759, 0x0A75B, 0x0A75B, 0x0A75D, 0x0A75D, 0x0A75F, 0x0A75F, - 0x0A761, 0x0A761, 0x0A763, 0x0A763, 0x0A765, 0x0A765, 0x0A767, 0x0A767, - 0x0A769, 0x0A769, 0x0A76B, 0x0A76B, 0x0A76D, 0x0A76D, 0x0A76F, 0x0A76F, - 0x0A770, 0x0A771, 0x0A772, 0x0A773, 0x0A774, 0x0A775, 0x0A776, 0x0A777, - 0x0A778, 0x0A77A, 0x0A77A, 0x0A77C, 0x0A77C, 0x01D79, 0x0A77F, 0x0A77F, -}; - - -static const uint32_t nxt_unicode_block_14f[128] nxt_aligned(64) = { - 0x0A781, 0x0A781, 0x0A783, 0x0A783, 0x0A785, 0x0A785, 0x0A787, 0x0A787, - 0x0A788, 0x0A789, 0x0A78A, 0x0A78C, 0x0A78C, 0x00265, 0x0A78E, 0x0A78F, - 0x0A791, 0x0A791, 0x0A793, 0x0A793, 0x0A794, 0x0A795, 0x0A796, 0x0A797, - 0x0A798, 0x0A799, 0x0A79A, 0x0A79B, 0x0A79C, 0x0A79D, 0x0A79E, 0x0A79F, - 0x0A7A1, 0x0A7A1, 0x0A7A3, 0x0A7A3, 0x0A7A5, 0x0A7A5, 0x0A7A7, 0x0A7A7, - 0x0A7A9, 0x0A7A9, 0x00266, 0x0A7AB, 0x0A7AC, 0x0A7AD, 0x0A7AE, 0x0A7AF, - 0x0A7B0, 0x0A7B1, 0x0A7B2, 0x0A7B3, 0x0A7B4, 0x0A7B5, 0x0A7B6, 0x0A7B7, - 0x0A7B8, 0x0A7B9, 0x0A7BA, 0x0A7BB, 0x0A7BC, 0x0A7BD, 0x0A7BE, 0x0A7BF, - 0x0A7C0, 0x0A7C1, 0x0A7C2, 0x0A7C3, 0x0A7C4, 0x0A7C5, 0x0A7C6, 0x0A7C7, - 0x0A7C8, 0x0A7C9, 0x0A7CA, 0x0A7CB, 0x0A7CC, 0x0A7CD, 0x0A7CE, 0x0A7CF, - 0x0A7D0, 0x0A7D1, 0x0A7D2, 0x0A7D3, 0x0A7D4, 0x0A7D5, 0x0A7D6, 0x0A7D7, - 0x0A7D8, 0x0A7D9, 0x0A7DA, 0x0A7DB, 0x0A7DC, 0x0A7DD, 0x0A7DE, 0x0A7DF, - 0x0A7E0, 0x0A7E1, 0x0A7E2, 0x0A7E3, 0x0A7E4, 0x0A7E5, 0x0A7E6, 0x0A7E7, - 0x0A7E8, 0x0A7E9, 0x0A7EA, 0x0A7EB, 0x0A7EC, 0x0A7ED, 0x0A7EE, 0x0A7EF, - 0x0A7F0, 0x0A7F1, 0x0A7F2, 0x0A7F3, 0x0A7F4, 0x0A7F5, 0x0A7F6, 0x0A7F7, - 0x0A7F8, 0x0A7F9, 0x0A7FA, 0x0A7FB, 0x0A7FC, 0x0A7FD, 0x0A7FE, 0x0A7FF, -}; - - -static const uint32_t nxt_unicode_block_1fe[128] nxt_aligned(64) = { - 0x0FF00, 0x0FF01, 0x0FF02, 0x0FF03, 0x0FF04, 0x0FF05, 0x0FF06, 0x0FF07, - 0x0FF08, 0x0FF09, 0x0FF0A, 0x0FF0B, 0x0FF0C, 0x0FF0D, 0x0FF0E, 0x0FF0F, - 0x0FF10, 0x0FF11, 0x0FF12, 0x0FF13, 0x0FF14, 0x0FF15, 0x0FF16, 0x0FF17, - 0x0FF18, 0x0FF19, 0x0FF1A, 0x0FF1B, 0x0FF1C, 0x0FF1D, 0x0FF1E, 0x0FF1F, - 0x0FF20, 0x0FF41, 0x0FF42, 0x0FF43, 0x0FF44, 0x0FF45, 0x0FF46, 0x0FF47, - 0x0FF48, 0x0FF49, 0x0FF4A, 0x0FF4B, 0x0FF4C, 0x0FF4D, 0x0FF4E, 0x0FF4F, - 0x0FF50, 0x0FF51, 0x0FF52, 0x0FF53, 0x0FF54, 0x0FF55, 0x0FF56, 0x0FF57, - 0x0FF58, 0x0FF59, 0x0FF5A, 0x0FF3B, 0x0FF3C, 0x0FF3D, 0x0FF3E, 0x0FF3F, - 0x0FF40, 0x0FF41, 0x0FF42, 0x0FF43, 0x0FF44, 0x0FF45, 0x0FF46, 0x0FF47, - 0x0FF48, 0x0FF49, 0x0FF4A, 0x0FF4B, 0x0FF4C, 0x0FF4D, 0x0FF4E, 0x0FF4F, - 0x0FF50, 0x0FF51, 0x0FF52, 0x0FF53, 0x0FF54, 0x0FF55, 0x0FF56, 0x0FF57, - 0x0FF58, 0x0FF59, 0x0FF5A, 0x0FF5B, 0x0FF5C, 0x0FF5D, 0x0FF5E, 0x0FF5F, - 0x0FF60, 0x0FF61, 0x0FF62, 0x0FF63, 0x0FF64, 0x0FF65, 0x0FF66, 0x0FF67, - 0x0FF68, 0x0FF69, 0x0FF6A, 0x0FF6B, 0x0FF6C, 0x0FF6D, 0x0FF6E, 0x0FF6F, - 0x0FF70, 0x0FF71, 0x0FF72, 0x0FF73, 0x0FF74, 0x0FF75, 0x0FF76, 0x0FF77, - 0x0FF78, 0x0FF79, 0x0FF7A, 0x0FF7B, 0x0FF7C, 0x0FF7D, 0x0FF7E, 0x0FF7F, -}; - - -static const uint32_t nxt_unicode_block_208[40] nxt_aligned(64) = { - 0x10428, 0x10429, 0x1042A, 0x1042B, 0x1042C, 0x1042D, 0x1042E, 0x1042F, - 0x10430, 0x10431, 0x10432, 0x10433, 0x10434, 0x10435, 0x10436, 0x10437, - 0x10438, 0x10439, 0x1043A, 0x1043B, 0x1043C, 0x1043D, 0x1043E, 0x1043F, - 0x10440, 0x10441, 0x10442, 0x10443, 0x10444, 0x10445, 0x10446, 0x10447, - 0x10448, 0x10449, 0x1044A, 0x1044B, 0x1044C, 0x1044D, 0x1044E, 0x1044F, -}; - - -static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = { - nxt_unicode_block_000, - nxt_unicode_block_001, - nxt_unicode_block_002, - nxt_unicode_block_003, - nxt_unicode_block_004, - NULL, - nxt_unicode_block_006, - nxt_unicode_block_007, - nxt_unicode_block_008, - nxt_unicode_block_009, - nxt_unicode_block_00a, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_021, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_03c, - nxt_unicode_block_03d, - nxt_unicode_block_03e, - nxt_unicode_block_03f, - NULL, - NULL, - nxt_unicode_block_042, - nxt_unicode_block_043, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_049, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_058, - nxt_unicode_block_059, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_14c, - nxt_unicode_block_14d, - nxt_unicode_block_14e, - nxt_unicode_block_14f, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_1fe, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_208, -}; diff --git a/src/nxt_unicode_lowcase.pl b/src/nxt_unicode_lowcase.pl deleted file mode 100644 index abf649651..000000000 --- a/src/nxt_unicode_lowcase.pl +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/perl - -use warnings; -use strict; - -# BLOCK_SIZE should be 128, 256, 512, etc. The value 128 provides -# the minimum memory footprint for both 32-bit and 64-bit platforms. -use constant BLOCK_SIZE => 128; - -my %lowcase; -my %blocks; -my $max_block = 0; -my $max_lowcase = 0; - -while (<>) { - if (/^(\w+); (C|S); (\w+);/) { - my ($symbol, $folding) = (hex $1, hex $3); - $lowcase{$symbol} = $folding; - $blocks{int($symbol / BLOCK_SIZE)} = 1; - - if ($max_lowcase < $symbol) { - $max_lowcase = $symbol; - } - } -} - - -my $last_block_size = $max_lowcase % BLOCK_SIZE + 1; - - -for my $block (sort { $a <=> $b } keys %blocks) { - if ($max_block < $block) { - $max_block = $block; - } -} - - -my $blocks = scalar keys %blocks; - -printf("\n/*\n" . - " * %d %s-bytes blocks, %d pointers.\n" . - " * %d bytes on 32-bit platforms, %d bytes on 64-bit platforms.\n" . - " */\n\n", - $blocks, BLOCK_SIZE, $max_block + 1, - ($blocks - 1) * BLOCK_SIZE * 4 + $last_block_size + $max_block * 4, - ($blocks - 1) * BLOCK_SIZE * 4 + $last_block_size+ $max_block * 8); - -printf("#define NXT_UNICODE_MAX_LOWCASE 0x%05x\n\n", $max_lowcase); -printf("#define NXT_UNICODE_BLOCK_SIZE %d\n\n\n", BLOCK_SIZE); - - -for my $block (sort { $a <=> $b } keys %blocks) { - my $block_size = ($block != $max_block) ? BLOCK_SIZE : $last_block_size; - - print "static const uint32_t "; - printf("nxt_unicode_block_%03x[%d] nxt_aligned(64) = {", - $block, $block_size); - - for my $c (0 .. $block_size - 1) { - printf "\n " if $c % 8 == 0; - - my $n = $block * BLOCK_SIZE + $c; - - if (exists $lowcase{$n}) { - printf(" 0x%05x,", $lowcase{$n}); - - } else { - #print " .......,"; - printf(" 0x%05x,", $n); - } - } - - print "\n};\n\n\n"; -} - - -print "static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = {\n"; - -for my $block (0 .. $max_block) { - if (exists($blocks{$block})) { - printf(" nxt_unicode_block_%03x,\n", $block); - - } else { - print " NULL,\n"; - } -} - -print "};\n"; diff --git a/src/nxt_unicode_macosx_lowcase.h b/src/nxt_unicode_macosx_lowcase.h deleted file mode 100644 index d0cc15186..000000000 --- a/src/nxt_unicode_macosx_lowcase.h +++ /dev/null @@ -1,816 +0,0 @@ - -/* - * 15 128-bytes blocks, 511 pointers. - * 9267 bytes on 32-bit platforms, 11307 bytes on 64-bit platforms. - */ - -#define NXT_UNICODE_MAX_LOWCASE 0x0FF3A - -#define NXT_UNICODE_BLOCK_SIZE 128 - - -static const uint32_t nxt_unicode_block_000[128] nxt_aligned(64) = { - 0x00000, 0x00001, 0x00002, 0x00003, 0x00004, 0x00005, 0x00006, 0x00007, - 0x00008, 0x00009, 0x0000A, 0x0000B, 0x0000C, 0x0000D, 0x0000E, 0x0000F, - 0x00010, 0x00011, 0x00012, 0x00013, 0x00014, 0x00015, 0x00016, 0x00017, - 0x00018, 0x00019, 0x0001A, 0x0001B, 0x0001C, 0x0001D, 0x0001E, 0x0001F, - 0x00020, 0x00021, 0x00022, 0x00023, 0x00024, 0x00025, 0x00026, 0x00027, - 0x00028, 0x00029, 0x0002A, 0x0002B, 0x0002C, 0x0002D, 0x0002E, 0x0002F, - 0x00030, 0x00031, 0x00032, 0x00033, 0x00034, 0x00035, 0x00036, 0x00037, - 0x00038, 0x00039, 0x0003A, 0x0003B, 0x0003C, 0x0003D, 0x0003E, 0x0003F, - 0x00040, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, - 0x00068, 0x00069, 0x0006A, 0x0006B, 0x0006C, 0x0006D, 0x0006E, 0x0006F, - 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, - 0x00078, 0x00079, 0x0007A, 0x0005B, 0x0005C, 0x0005D, 0x0005E, 0x0005F, - 0x00060, 0x00061, 0x00062, 0x00063, 0x00064, 0x00065, 0x00066, 0x00067, - 0x00068, 0x00069, 0x0006A, 0x0006B, 0x0006C, 0x0006D, 0x0006E, 0x0006F, - 0x00070, 0x00071, 0x00072, 0x00073, 0x00074, 0x00075, 0x00076, 0x00077, - 0x00078, 0x00079, 0x0007A, 0x0007B, 0x0007C, 0x0007D, 0x0007E, 0x0007F, -}; - - -static const uint32_t nxt_unicode_block_001[128] nxt_aligned(64) = { - 0x00080, 0x00081, 0x00082, 0x00083, 0x00084, 0x00085, 0x00086, 0x00087, - 0x00088, 0x00089, 0x0008A, 0x0008B, 0x0008C, 0x0008D, 0x0008E, 0x0008F, - 0x00090, 0x00091, 0x00092, 0x00093, 0x00094, 0x00095, 0x00096, 0x00097, - 0x00098, 0x00099, 0x0009A, 0x0009B, 0x0009C, 0x0009D, 0x0009E, 0x0009F, - 0x000A0, 0x000A1, 0x000A2, 0x000A3, 0x000A4, 0x000A5, 0x000A6, 0x000A7, - 0x000A8, 0x000A9, 0x000AA, 0x000AB, 0x000AC, 0x000AD, 0x000AE, 0x000AF, - 0x000B0, 0x000B1, 0x000B2, 0x000B3, 0x000B4, 0x000B5, 0x000B6, 0x000B7, - 0x000B8, 0x000B9, 0x000BA, 0x000BB, 0x000BC, 0x000BD, 0x000BE, 0x000BF, - 0x000E0, 0x000E1, 0x000E2, 0x000E3, 0x000E4, 0x000E5, 0x000E6, 0x000E7, - 0x000E8, 0x000E9, 0x000EA, 0x000EB, 0x000EC, 0x000ED, 0x000EE, 0x000EF, - 0x000F0, 0x000F1, 0x000F2, 0x000F3, 0x000F4, 0x000F5, 0x000F6, 0x000D7, - 0x000F8, 0x000F9, 0x000FA, 0x000FB, 0x000FC, 0x000FD, 0x000FE, 0x000DF, - 0x000E0, 0x000E1, 0x000E2, 0x000E3, 0x000E4, 0x000E5, 0x000E6, 0x000E7, - 0x000E8, 0x000E9, 0x000EA, 0x000EB, 0x000EC, 0x000ED, 0x000EE, 0x000EF, - 0x000F0, 0x000F1, 0x000F2, 0x000F3, 0x000F4, 0x000F5, 0x000F6, 0x000F7, - 0x000F8, 0x000F9, 0x000FA, 0x000FB, 0x000FC, 0x000FD, 0x000FE, 0x000FF, -}; - - -static const uint32_t nxt_unicode_block_002[128] nxt_aligned(64) = { - 0x00101, 0x00101, 0x00103, 0x00103, 0x00105, 0x00105, 0x00107, 0x00107, - 0x00109, 0x00109, 0x0010B, 0x0010B, 0x0010D, 0x0010D, 0x0010F, 0x0010F, - 0x00111, 0x00111, 0x00113, 0x00113, 0x00115, 0x00115, 0x00117, 0x00117, - 0x00119, 0x00119, 0x0011B, 0x0011B, 0x0011D, 0x0011D, 0x0011F, 0x0011F, - 0x00121, 0x00121, 0x00123, 0x00123, 0x00125, 0x00125, 0x00127, 0x00127, - 0x00129, 0x00129, 0x0012B, 0x0012B, 0x0012D, 0x0012D, 0x0012F, 0x0012F, - 0x00130, 0x00131, 0x00133, 0x00133, 0x00135, 0x00135, 0x00137, 0x00137, - 0x00138, 0x0013A, 0x0013A, 0x0013C, 0x0013C, 0x0013E, 0x0013E, 0x00140, - 0x00140, 0x00142, 0x00142, 0x00144, 0x00144, 0x00146, 0x00146, 0x00148, - 0x00148, 0x00149, 0x0014B, 0x0014B, 0x0014D, 0x0014D, 0x0014F, 0x0014F, - 0x00151, 0x00151, 0x00153, 0x00153, 0x00155, 0x00155, 0x00157, 0x00157, - 0x00159, 0x00159, 0x0015B, 0x0015B, 0x0015D, 0x0015D, 0x0015F, 0x0015F, - 0x00161, 0x00161, 0x00163, 0x00163, 0x00165, 0x00165, 0x00167, 0x00167, - 0x00169, 0x00169, 0x0016B, 0x0016B, 0x0016D, 0x0016D, 0x0016F, 0x0016F, - 0x00171, 0x00171, 0x00173, 0x00173, 0x00175, 0x00175, 0x00177, 0x00177, - 0x000FF, 0x0017A, 0x0017A, 0x0017C, 0x0017C, 0x0017E, 0x0017E, 0x0017F, -}; - - -static const uint32_t nxt_unicode_block_003[128] nxt_aligned(64) = { - 0x00180, 0x00253, 0x00183, 0x00183, 0x00185, 0x00185, 0x00254, 0x00188, - 0x00188, 0x00256, 0x00257, 0x0018C, 0x0018C, 0x0018D, 0x001DD, 0x00259, - 0x0025B, 0x00192, 0x00192, 0x00260, 0x00263, 0x00195, 0x00269, 0x00268, - 0x00199, 0x00199, 0x0019A, 0x0019B, 0x0026F, 0x00272, 0x0019E, 0x00275, - 0x001A1, 0x001A1, 0x001A3, 0x001A3, 0x001A5, 0x001A5, 0x001A6, 0x001A8, - 0x001A8, 0x00283, 0x001AA, 0x001AB, 0x001AD, 0x001AD, 0x00288, 0x001B0, - 0x001B0, 0x0028A, 0x0028B, 0x001B4, 0x001B4, 0x001B6, 0x001B6, 0x00292, - 0x001B9, 0x001B9, 0x001BA, 0x001BB, 0x001BD, 0x001BD, 0x001BE, 0x001BF, - 0x001C0, 0x001C1, 0x001C2, 0x001C3, 0x001C6, 0x001C6, 0x001C6, 0x001C9, - 0x001C9, 0x001C9, 0x001CC, 0x001CC, 0x001CC, 0x001CE, 0x001CE, 0x001D0, - 0x001D0, 0x001D2, 0x001D2, 0x001D4, 0x001D4, 0x001D6, 0x001D6, 0x001D8, - 0x001D8, 0x001DA, 0x001DA, 0x001DC, 0x001DC, 0x001DD, 0x001DF, 0x001DF, - 0x001E1, 0x001E1, 0x001E3, 0x001E3, 0x001E5, 0x001E5, 0x001E7, 0x001E7, - 0x001E9, 0x001E9, 0x001EB, 0x001EB, 0x001ED, 0x001ED, 0x001EF, 0x001EF, - 0x001F0, 0x001F3, 0x001F3, 0x001F3, 0x001F5, 0x001F5, 0x001F6, 0x001F7, - 0x001F9, 0x001F9, 0x001FB, 0x001FB, 0x001FD, 0x001FD, 0x001FF, 0x001FF, -}; - - -static const uint32_t nxt_unicode_block_004[128] nxt_aligned(64) = { - 0x00201, 0x00201, 0x00203, 0x00203, 0x00205, 0x00205, 0x00207, 0x00207, - 0x00209, 0x00209, 0x0020B, 0x0020B, 0x0020D, 0x0020D, 0x0020F, 0x0020F, - 0x00211, 0x00211, 0x00213, 0x00213, 0x00215, 0x00215, 0x00217, 0x00217, - 0x00219, 0x00219, 0x0021B, 0x0021B, 0x0021C, 0x0021D, 0x0021F, 0x0021F, - 0x00220, 0x00221, 0x00222, 0x00223, 0x00224, 0x00225, 0x00227, 0x00227, - 0x00229, 0x00229, 0x0022B, 0x0022B, 0x0022D, 0x0022D, 0x0022F, 0x0022F, - 0x00231, 0x00231, 0x00233, 0x00233, 0x00234, 0x00235, 0x00236, 0x00237, - 0x00238, 0x00239, 0x0023A, 0x0023B, 0x0023C, 0x0023D, 0x0023E, 0x0023F, - 0x00240, 0x00241, 0x00242, 0x00243, 0x00244, 0x00245, 0x00246, 0x00247, - 0x00248, 0x00249, 0x0024A, 0x0024B, 0x0024C, 0x0024D, 0x0024E, 0x0024F, - 0x00250, 0x00251, 0x00252, 0x00253, 0x00254, 0x00255, 0x00256, 0x00257, - 0x00258, 0x00259, 0x0025A, 0x0025B, 0x0025C, 0x0025D, 0x0025E, 0x0025F, - 0x00260, 0x00261, 0x00262, 0x00263, 0x00264, 0x00265, 0x00266, 0x00267, - 0x00268, 0x00269, 0x0026A, 0x0026B, 0x0026C, 0x0026D, 0x0026E, 0x0026F, - 0x00270, 0x00271, 0x00272, 0x00273, 0x00274, 0x00275, 0x00276, 0x00277, - 0x00278, 0x00279, 0x0027A, 0x0027B, 0x0027C, 0x0027D, 0x0027E, 0x0027F, -}; - - -static const uint32_t nxt_unicode_block_007[128] nxt_aligned(64) = { - 0x00380, 0x00381, 0x00382, 0x00383, 0x00384, 0x00385, 0x003AC, 0x00387, - 0x003AD, 0x003AE, 0x003AF, 0x0038B, 0x003CC, 0x0038D, 0x003CD, 0x003CE, - 0x00390, 0x003B1, 0x003B2, 0x003B3, 0x003B4, 0x003B5, 0x003B6, 0x003B7, - 0x003B8, 0x003B9, 0x003BA, 0x003BB, 0x003BC, 0x003BD, 0x003BE, 0x003BF, - 0x003C0, 0x003C1, 0x003A2, 0x003C3, 0x003C4, 0x003C5, 0x003C6, 0x003C7, - 0x003C8, 0x003C9, 0x003CA, 0x003CB, 0x003AC, 0x003AD, 0x003AE, 0x003AF, - 0x003B0, 0x003B1, 0x003B2, 0x003B3, 0x003B4, 0x003B5, 0x003B6, 0x003B7, - 0x003B8, 0x003B9, 0x003BA, 0x003BB, 0x003BC, 0x003BD, 0x003BE, 0x003BF, - 0x003C0, 0x003C1, 0x003C2, 0x003C3, 0x003C4, 0x003C5, 0x003C6, 0x003C7, - 0x003C8, 0x003C9, 0x003CA, 0x003CB, 0x003CC, 0x003CD, 0x003CE, 0x003CF, - 0x003D0, 0x003D1, 0x003D2, 0x003D3, 0x003D4, 0x003D5, 0x003D6, 0x003D7, - 0x003D8, 0x003D9, 0x003DA, 0x003DB, 0x003DC, 0x003DD, 0x003DE, 0x003DF, - 0x003E0, 0x003E1, 0x003E3, 0x003E3, 0x003E5, 0x003E5, 0x003E7, 0x003E7, - 0x003E9, 0x003E9, 0x003EB, 0x003EB, 0x003ED, 0x003ED, 0x003EF, 0x003EF, - 0x003F0, 0x003F1, 0x003F2, 0x003F3, 0x003F4, 0x003F5, 0x003F6, 0x003F7, - 0x003F8, 0x003F9, 0x003FA, 0x003FB, 0x003FC, 0x003FD, 0x003FE, 0x003FF, -}; - - -static const uint32_t nxt_unicode_block_008[128] nxt_aligned(64) = { - 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, - 0x00458, 0x00459, 0x0045A, 0x0045B, 0x0045C, 0x0045D, 0x0045E, 0x0045F, - 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, - 0x00438, 0x00439, 0x0043A, 0x0043B, 0x0043C, 0x0043D, 0x0043E, 0x0043F, - 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, - 0x00448, 0x00449, 0x0044A, 0x0044B, 0x0044C, 0x0044D, 0x0044E, 0x0044F, - 0x00430, 0x00431, 0x00432, 0x00433, 0x00434, 0x00435, 0x00436, 0x00437, - 0x00438, 0x00439, 0x0043A, 0x0043B, 0x0043C, 0x0043D, 0x0043E, 0x0043F, - 0x00440, 0x00441, 0x00442, 0x00443, 0x00444, 0x00445, 0x00446, 0x00447, - 0x00448, 0x00449, 0x0044A, 0x0044B, 0x0044C, 0x0044D, 0x0044E, 0x0044F, - 0x00450, 0x00451, 0x00452, 0x00453, 0x00454, 0x00455, 0x00456, 0x00457, - 0x00458, 0x00459, 0x0045A, 0x0045B, 0x0045C, 0x0045D, 0x0045E, 0x0045F, - 0x00461, 0x00461, 0x00463, 0x00463, 0x00465, 0x00465, 0x00467, 0x00467, - 0x00469, 0x00469, 0x0046B, 0x0046B, 0x0046D, 0x0046D, 0x0046F, 0x0046F, - 0x00471, 0x00471, 0x00473, 0x00473, 0x00475, 0x00475, 0x00477, 0x00477, - 0x00479, 0x00479, 0x0047B, 0x0047B, 0x0047D, 0x0047D, 0x0047F, 0x0047F, -}; - - -static const uint32_t nxt_unicode_block_009[128] nxt_aligned(64) = { - 0x00481, 0x00481, 0x00482, 0x00483, 0x00484, 0x00485, 0x00486, 0x00487, - 0x00488, 0x00489, 0x0048A, 0x0048B, 0x0048C, 0x0048D, 0x0048E, 0x0048F, - 0x00491, 0x00491, 0x00493, 0x00493, 0x00495, 0x00495, 0x00497, 0x00497, - 0x00499, 0x00499, 0x0049B, 0x0049B, 0x0049D, 0x0049D, 0x0049F, 0x0049F, - 0x004A1, 0x004A1, 0x004A3, 0x004A3, 0x004A5, 0x004A5, 0x004A7, 0x004A7, - 0x004A9, 0x004A9, 0x004AB, 0x004AB, 0x004AD, 0x004AD, 0x004AF, 0x004AF, - 0x004B1, 0x004B1, 0x004B3, 0x004B3, 0x004B5, 0x004B5, 0x004B7, 0x004B7, - 0x004B9, 0x004B9, 0x004BB, 0x004BB, 0x004BD, 0x004BD, 0x004BF, 0x004BF, - 0x004C0, 0x004C2, 0x004C2, 0x004C4, 0x004C4, 0x004C5, 0x004C6, 0x004C8, - 0x004C8, 0x004C9, 0x004CA, 0x004CC, 0x004CC, 0x004CD, 0x004CE, 0x004CF, - 0x004D1, 0x004D1, 0x004D3, 0x004D3, 0x004D4, 0x004D5, 0x004D7, 0x004D7, - 0x004D8, 0x004D9, 0x004DA, 0x004DB, 0x004DD, 0x004DD, 0x004DF, 0x004DF, - 0x004E0, 0x004E1, 0x004E3, 0x004E3, 0x004E5, 0x004E5, 0x004E7, 0x004E7, - 0x004E8, 0x004E9, 0x004EA, 0x004EB, 0x004ED, 0x004ED, 0x004EF, 0x004EF, - 0x004F1, 0x004F1, 0x004F3, 0x004F3, 0x004F5, 0x004F5, 0x004F6, 0x004F7, - 0x004F9, 0x004F9, 0x004FA, 0x004FB, 0x004FC, 0x004FD, 0x004FE, 0x004FF, -}; - - -static const uint32_t nxt_unicode_block_00a[128] nxt_aligned(64) = { - 0x00500, 0x00501, 0x00502, 0x00503, 0x00504, 0x00505, 0x00506, 0x00507, - 0x00508, 0x00509, 0x0050A, 0x0050B, 0x0050C, 0x0050D, 0x0050E, 0x0050F, - 0x00510, 0x00511, 0x00512, 0x00513, 0x00514, 0x00515, 0x00516, 0x00517, - 0x00518, 0x00519, 0x0051A, 0x0051B, 0x0051C, 0x0051D, 0x0051E, 0x0051F, - 0x00520, 0x00521, 0x00522, 0x00523, 0x00524, 0x00525, 0x00526, 0x00527, - 0x00528, 0x00529, 0x0052A, 0x0052B, 0x0052C, 0x0052D, 0x0052E, 0x0052F, - 0x00530, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, - 0x00568, 0x00569, 0x0056A, 0x0056B, 0x0056C, 0x0056D, 0x0056E, 0x0056F, - 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, - 0x00578, 0x00579, 0x0057A, 0x0057B, 0x0057C, 0x0057D, 0x0057E, 0x0057F, - 0x00580, 0x00581, 0x00582, 0x00583, 0x00584, 0x00585, 0x00586, 0x00557, - 0x00558, 0x00559, 0x0055A, 0x0055B, 0x0055C, 0x0055D, 0x0055E, 0x0055F, - 0x00560, 0x00561, 0x00562, 0x00563, 0x00564, 0x00565, 0x00566, 0x00567, - 0x00568, 0x00569, 0x0056A, 0x0056B, 0x0056C, 0x0056D, 0x0056E, 0x0056F, - 0x00570, 0x00571, 0x00572, 0x00573, 0x00574, 0x00575, 0x00576, 0x00577, - 0x00578, 0x00579, 0x0057A, 0x0057B, 0x0057C, 0x0057D, 0x0057E, 0x0057F, -}; - - -static const uint32_t nxt_unicode_block_03c[128] nxt_aligned(64) = { - 0x01E01, 0x01E01, 0x01E03, 0x01E03, 0x01E05, 0x01E05, 0x01E07, 0x01E07, - 0x01E09, 0x01E09, 0x01E0B, 0x01E0B, 0x01E0D, 0x01E0D, 0x01E0F, 0x01E0F, - 0x01E11, 0x01E11, 0x01E13, 0x01E13, 0x01E15, 0x01E15, 0x01E17, 0x01E17, - 0x01E19, 0x01E19, 0x01E1B, 0x01E1B, 0x01E1D, 0x01E1D, 0x01E1F, 0x01E1F, - 0x01E21, 0x01E21, 0x01E23, 0x01E23, 0x01E25, 0x01E25, 0x01E27, 0x01E27, - 0x01E29, 0x01E29, 0x01E2B, 0x01E2B, 0x01E2D, 0x01E2D, 0x01E2F, 0x01E2F, - 0x01E31, 0x01E31, 0x01E33, 0x01E33, 0x01E35, 0x01E35, 0x01E37, 0x01E37, - 0x01E39, 0x01E39, 0x01E3B, 0x01E3B, 0x01E3D, 0x01E3D, 0x01E3F, 0x01E3F, - 0x01E41, 0x01E41, 0x01E43, 0x01E43, 0x01E45, 0x01E45, 0x01E47, 0x01E47, - 0x01E49, 0x01E49, 0x01E4B, 0x01E4B, 0x01E4D, 0x01E4D, 0x01E4F, 0x01E4F, - 0x01E51, 0x01E51, 0x01E53, 0x01E53, 0x01E55, 0x01E55, 0x01E57, 0x01E57, - 0x01E59, 0x01E59, 0x01E5B, 0x01E5B, 0x01E5D, 0x01E5D, 0x01E5F, 0x01E5F, - 0x01E61, 0x01E61, 0x01E63, 0x01E63, 0x01E65, 0x01E65, 0x01E67, 0x01E67, - 0x01E69, 0x01E69, 0x01E6B, 0x01E6B, 0x01E6D, 0x01E6D, 0x01E6F, 0x01E6F, - 0x01E71, 0x01E71, 0x01E73, 0x01E73, 0x01E75, 0x01E75, 0x01E77, 0x01E77, - 0x01E79, 0x01E79, 0x01E7B, 0x01E7B, 0x01E7D, 0x01E7D, 0x01E7F, 0x01E7F, -}; - - -static const uint32_t nxt_unicode_block_03d[128] nxt_aligned(64) = { - 0x01E81, 0x01E81, 0x01E83, 0x01E83, 0x01E85, 0x01E85, 0x01E87, 0x01E87, - 0x01E89, 0x01E89, 0x01E8B, 0x01E8B, 0x01E8D, 0x01E8D, 0x01E8F, 0x01E8F, - 0x01E91, 0x01E91, 0x01E93, 0x01E93, 0x01E95, 0x01E95, 0x01E96, 0x01E97, - 0x01E98, 0x01E99, 0x01E9A, 0x01E9B, 0x01E9C, 0x01E9D, 0x01E9E, 0x01E9F, - 0x01EA1, 0x01EA1, 0x01EA3, 0x01EA3, 0x01EA5, 0x01EA5, 0x01EA7, 0x01EA7, - 0x01EA9, 0x01EA9, 0x01EAB, 0x01EAB, 0x01EAD, 0x01EAD, 0x01EAF, 0x01EAF, - 0x01EB1, 0x01EB1, 0x01EB3, 0x01EB3, 0x01EB5, 0x01EB5, 0x01EB7, 0x01EB7, - 0x01EB9, 0x01EB9, 0x01EBB, 0x01EBB, 0x01EBD, 0x01EBD, 0x01EBF, 0x01EBF, - 0x01EC1, 0x01EC1, 0x01EC3, 0x01EC3, 0x01EC5, 0x01EC5, 0x01EC7, 0x01EC7, - 0x01EC9, 0x01EC9, 0x01ECB, 0x01ECB, 0x01ECD, 0x01ECD, 0x01ECF, 0x01ECF, - 0x01ED1, 0x01ED1, 0x01ED3, 0x01ED3, 0x01ED5, 0x01ED5, 0x01ED7, 0x01ED7, - 0x01ED9, 0x01ED9, 0x01EDB, 0x01EDB, 0x01EDD, 0x01EDD, 0x01EDF, 0x01EDF, - 0x01EE1, 0x01EE1, 0x01EE3, 0x01EE3, 0x01EE5, 0x01EE5, 0x01EE7, 0x01EE7, - 0x01EE9, 0x01EE9, 0x01EEB, 0x01EEB, 0x01EED, 0x01EED, 0x01EEF, 0x01EEF, - 0x01EF1, 0x01EF1, 0x01EF3, 0x01EF3, 0x01EF5, 0x01EF5, 0x01EF7, 0x01EF7, - 0x01EF9, 0x01EF9, 0x01EFA, 0x01EFB, 0x01EFC, 0x01EFD, 0x01EFE, 0x01EFF, -}; - - -static const uint32_t nxt_unicode_block_03e[128] nxt_aligned(64) = { - 0x01F00, 0x01F01, 0x01F02, 0x01F03, 0x01F04, 0x01F05, 0x01F06, 0x01F07, - 0x01F00, 0x01F01, 0x01F02, 0x01F03, 0x01F04, 0x01F05, 0x01F06, 0x01F07, - 0x01F10, 0x01F11, 0x01F12, 0x01F13, 0x01F14, 0x01F15, 0x01F16, 0x01F17, - 0x01F10, 0x01F11, 0x01F12, 0x01F13, 0x01F14, 0x01F15, 0x01F1E, 0x01F1F, - 0x01F20, 0x01F21, 0x01F22, 0x01F23, 0x01F24, 0x01F25, 0x01F26, 0x01F27, - 0x01F20, 0x01F21, 0x01F22, 0x01F23, 0x01F24, 0x01F25, 0x01F26, 0x01F27, - 0x01F30, 0x01F31, 0x01F32, 0x01F33, 0x01F34, 0x01F35, 0x01F36, 0x01F37, - 0x01F30, 0x01F31, 0x01F32, 0x01F33, 0x01F34, 0x01F35, 0x01F36, 0x01F37, - 0x01F40, 0x01F41, 0x01F42, 0x01F43, 0x01F44, 0x01F45, 0x01F46, 0x01F47, - 0x01F40, 0x01F41, 0x01F42, 0x01F43, 0x01F44, 0x01F45, 0x01F4E, 0x01F4F, - 0x01F50, 0x01F51, 0x01F52, 0x01F53, 0x01F54, 0x01F55, 0x01F56, 0x01F57, - 0x01F58, 0x01F51, 0x01F5A, 0x01F53, 0x01F5C, 0x01F55, 0x01F5E, 0x01F57, - 0x01F60, 0x01F61, 0x01F62, 0x01F63, 0x01F64, 0x01F65, 0x01F66, 0x01F67, - 0x01F60, 0x01F61, 0x01F62, 0x01F63, 0x01F64, 0x01F65, 0x01F66, 0x01F67, - 0x01F70, 0x01F71, 0x01F72, 0x01F73, 0x01F74, 0x01F75, 0x01F76, 0x01F77, - 0x01F78, 0x01F79, 0x01F7A, 0x01F7B, 0x01F7C, 0x01F7D, 0x01F7E, 0x01F7F, -}; - - -static const uint32_t nxt_unicode_block_03f[128] nxt_aligned(64) = { - 0x01F80, 0x01F81, 0x01F82, 0x01F83, 0x01F84, 0x01F85, 0x01F86, 0x01F87, - 0x01F80, 0x01F81, 0x01F82, 0x01F83, 0x01F84, 0x01F85, 0x01F86, 0x01F87, - 0x01F90, 0x01F91, 0x01F92, 0x01F93, 0x01F94, 0x01F95, 0x01F96, 0x01F97, - 0x01F90, 0x01F91, 0x01F92, 0x01F93, 0x01F94, 0x01F95, 0x01F96, 0x01F97, - 0x01FA0, 0x01FA1, 0x01FA2, 0x01FA3, 0x01FA4, 0x01FA5, 0x01FA6, 0x01FA7, - 0x01FA0, 0x01FA1, 0x01FA2, 0x01FA3, 0x01FA4, 0x01FA5, 0x01FA6, 0x01FA7, - 0x01FB0, 0x01FB1, 0x01FB2, 0x01FB3, 0x01FB4, 0x01FB5, 0x01FB6, 0x01FB7, - 0x01FB0, 0x01FB1, 0x01F70, 0x01F71, 0x01FB3, 0x01FBD, 0x003B9, 0x01FBF, - 0x01FC0, 0x01FC1, 0x01FC2, 0x01FC3, 0x01FC4, 0x01FC5, 0x01FC6, 0x01FC7, - 0x01F72, 0x01F73, 0x01F74, 0x01F75, 0x01FC3, 0x01FCD, 0x01FCE, 0x01FCF, - 0x01FD0, 0x01FD1, 0x01FD2, 0x01FD3, 0x01FD4, 0x01FD5, 0x01FD6, 0x01FD7, - 0x01FD0, 0x01FD1, 0x01F76, 0x01F77, 0x01FDC, 0x01FDD, 0x01FDE, 0x01FDF, - 0x01FE0, 0x01FE1, 0x01FE2, 0x01FE3, 0x01FE4, 0x01FE5, 0x01FE6, 0x01FE7, - 0x01FE0, 0x01FE1, 0x01F7A, 0x01F7B, 0x01FE5, 0x01FED, 0x01FEE, 0x01FEF, - 0x01FF0, 0x01FF1, 0x01FF2, 0x01FF3, 0x01FF4, 0x01FF5, 0x01FF6, 0x01FF7, - 0x01F78, 0x01F79, 0x01F7C, 0x01F7D, 0x01FF3, 0x01FFD, 0x01FFE, 0x01FFF, -}; - - -static const uint32_t nxt_unicode_block_042[128] nxt_aligned(64) = { - 0x02100, 0x02101, 0x02102, 0x02103, 0x02104, 0x02105, 0x02106, 0x02107, - 0x02108, 0x02109, 0x0210A, 0x0210B, 0x0210C, 0x0210D, 0x0210E, 0x0210F, - 0x02110, 0x02111, 0x02112, 0x02113, 0x02114, 0x02115, 0x02116, 0x02117, - 0x02118, 0x02119, 0x0211A, 0x0211B, 0x0211C, 0x0211D, 0x0211E, 0x0211F, - 0x02120, 0x02121, 0x02122, 0x02123, 0x02124, 0x02125, 0x02126, 0x02127, - 0x02128, 0x02129, 0x0212A, 0x0212B, 0x0212C, 0x0212D, 0x0212E, 0x0212F, - 0x02130, 0x02131, 0x02132, 0x02133, 0x02134, 0x02135, 0x02136, 0x02137, - 0x02138, 0x02139, 0x0213A, 0x0213B, 0x0213C, 0x0213D, 0x0213E, 0x0213F, - 0x02140, 0x02141, 0x02142, 0x02143, 0x02144, 0x02145, 0x02146, 0x02147, - 0x02148, 0x02149, 0x0214A, 0x0214B, 0x0214C, 0x0214D, 0x0214E, 0x0214F, - 0x02150, 0x02151, 0x02152, 0x02153, 0x02154, 0x02155, 0x02156, 0x02157, - 0x02158, 0x02159, 0x0215A, 0x0215B, 0x0215C, 0x0215D, 0x0215E, 0x0215F, - 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, - 0x02178, 0x02179, 0x0217A, 0x0217B, 0x0217C, 0x0217D, 0x0217E, 0x0217F, - 0x02170, 0x02171, 0x02172, 0x02173, 0x02174, 0x02175, 0x02176, 0x02177, - 0x02178, 0x02179, 0x0217A, 0x0217B, 0x0217C, 0x0217D, 0x0217E, 0x0217F, -}; - - -static const uint32_t nxt_unicode_block_1fe[59] nxt_aligned(64) = { - 0x0FF00, 0x0FF01, 0x0FF02, 0x0FF03, 0x0FF04, 0x0FF05, 0x0FF06, 0x0FF07, - 0x0FF08, 0x0FF09, 0x0FF0A, 0x0FF0B, 0x0FF0C, 0x0FF0D, 0x0FF0E, 0x0FF0F, - 0x0FF10, 0x0FF11, 0x0FF12, 0x0FF13, 0x0FF14, 0x0FF15, 0x0FF16, 0x0FF17, - 0x0FF18, 0x0FF19, 0x0FF1A, 0x0FF1B, 0x0FF1C, 0x0FF1D, 0x0FF1E, 0x0FF1F, - 0x0FF20, 0x0FF41, 0x0FF42, 0x0FF43, 0x0FF44, 0x0FF45, 0x0FF46, 0x0FF47, - 0x0FF48, 0x0FF49, 0x0FF4A, 0x0FF4B, 0x0FF4C, 0x0FF4D, 0x0FF4E, 0x0FF4F, - 0x0FF50, 0x0FF51, 0x0FF52, 0x0FF53, 0x0FF54, 0x0FF55, 0x0FF56, 0x0FF57, - 0x0FF58, 0x0FF59, 0x0FF5A, -}; - - -static const uint32_t *nxt_unicode_blocks[] nxt_aligned(64) = { - nxt_unicode_block_000, - nxt_unicode_block_001, - nxt_unicode_block_002, - nxt_unicode_block_003, - nxt_unicode_block_004, - NULL, - NULL, - nxt_unicode_block_007, - nxt_unicode_block_008, - nxt_unicode_block_009, - nxt_unicode_block_00a, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_03c, - nxt_unicode_block_03d, - nxt_unicode_block_03e, - nxt_unicode_block_03f, - NULL, - NULL, - nxt_unicode_block_042, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - nxt_unicode_block_1fe, -}; diff --git a/src/nxt_utf8.c b/src/nxt_utf8.c index ce3372a00..b4179eaa6 100644 --- a/src/nxt_utf8.c +++ b/src/nxt_utf8.c @@ -6,28 +6,8 @@ #include -/* - * The nxt_unicode_lowcase.h file is the auto-generated file from - * the CaseFolding-6.3.0.txt file provided by Unicode, Inc.: - * - * ./lib/src/nxt_unicode_lowcase.pl CaseFolding-6.3.0.txt - * - * This file should be copied to system specific nxt_unicode_SYSTEM_lowcase.h - * file and utf8_file_name_test should be built with this file. - * Then a correct system specific file should be generated: - * - * ./build/utf8_file_name_test | ./lib/src/nxt_unicode_lowcase.pl - * - * Only common and simple case foldings are supported. Full case foldings - * is not supported. Combined characters are also not supported. - */ -#if (NXT_MACOSX) -#include - -#else -#include -#endif +static uint32_t nxt_utf8_decode2(const u_char **start, const u_char *end); u_char * @@ -91,7 +71,7 @@ nxt_utf8_decode(const u_char **start, const u_char *end) * invalid or overlong UTF-8 sequence. */ -uint32_t +static uint32_t nxt_utf8_decode2(const u_char **start, const u_char *end) { u_char c; @@ -179,72 +159,6 @@ nxt_utf8_decode2(const u_char **start, const u_char *end) } -/* - * nxt_utf8_casecmp() tests only up to the minimum of given lengths, but - * requires lengths of both strings because otherwise nxt_utf8_decode2() - * may fail due to incomplete sequence. - */ - -nxt_int_t -nxt_utf8_casecmp(const u_char *start1, const u_char *start2, size_t len1, - size_t len2) -{ - int32_t n; - uint32_t u1, u2; - const u_char *end1, *end2; - - end1 = start1 + len1; - end2 = start2 + len2; - - while (start1 < end1 && start2 < end2) { - - u1 = nxt_utf8_lowcase(&start1, end1); - - u2 = nxt_utf8_lowcase(&start2, end2); - - if (nxt_slow_path((u1 | u2) == 0xFFFFFFFF)) { - return NXT_UTF8_SORT_INVALID; - } - - n = u1 - u2; - - if (n != 0) { - return (nxt_int_t) n; - } - } - - return 0; -} - - -uint32_t -nxt_utf8_lowcase(const u_char **start, const u_char *end) -{ - uint32_t u; - const uint32_t *block; - - u = (uint32_t) **start; - - if (nxt_fast_path(u < 0x80)) { - (*start)++; - - return nxt_unicode_block_000[u]; - } - - u = nxt_utf8_decode2(start, end); - - if (u <= NXT_UNICODE_MAX_LOWCASE) { - block = nxt_unicode_blocks[u / NXT_UNICODE_BLOCK_SIZE]; - - if (block != NULL) { - return block[u % NXT_UNICODE_BLOCK_SIZE]; - } - } - - return u; -} - - ssize_t nxt_utf8_length(const u_char *p, size_t len) { diff --git a/src/nxt_utf8.h b/src/nxt_utf8.h index c21317fda..2b581a70e 100644 --- a/src/nxt_utf8.h +++ b/src/nxt_utf8.h @@ -1,4 +1,3 @@ - /* * Copyright (C) Igor Sysoev * Copyright (C) NGINX, Inc. @@ -8,55 +7,12 @@ #define _NXT_UTF8_H_INCLUDED_ -/* - * Since the maximum valid Unicode character is 0x0010FFFF, the maximum - * difference between Unicode characters is lesser 0x0010FFFF and - * 0x0EEE0EEE can be used as value to indicate UTF-8 encoding error. - */ -#define NXT_UTF8_SORT_INVALID 0x0EEE0EEE - - NXT_EXPORT u_char *nxt_utf8_encode(u_char *p, uint32_t u); NXT_EXPORT nxt_int_t nxt_utf8_sanitize(nxt_mp_t *mp, nxt_str_t *dst, const nxt_str_t *src); NXT_EXPORT uint32_t nxt_utf8_decode(const u_char **start, const u_char *end); -NXT_EXPORT uint32_t nxt_utf8_decode2(const u_char **start, const u_char *end); -NXT_EXPORT nxt_int_t nxt_utf8_casecmp(const u_char *start1, - const u_char *start2, size_t len1, size_t len2); -NXT_EXPORT uint32_t nxt_utf8_lowcase(const u_char **start, const u_char *end); NXT_EXPORT ssize_t nxt_utf8_length(const u_char *p, size_t len); NXT_EXPORT nxt_bool_t nxt_utf8_is_valid(const u_char *p, size_t len); -/* nxt_utf8_next() expects a valid UTF-8 string. */ - -nxt_inline const u_char * -nxt_utf8_next(const u_char *p, const u_char *end) -{ - u_char c; - - c = *p++; - - if ((c & 0x80) != 0) { - - do { - /* - * The first UTF-8 byte is either 0xxxxxxx or 11xxxxxx. - * The next UTF-8 bytes are 10xxxxxx. - */ - c = *p; - - if ((c & 0xC0) != 0x80) { - return p; - } - - p++; - - } while (p < end); - } - - return p; -} - - #endif /* _NXT_UTF8_H_INCLUDED_ */ diff --git a/src/test/nxt_utf8_file_name_test.c b/src/test/nxt_utf8_file_name_test.c deleted file mode 100644 index b77cd4ab0..000000000 --- a/src/test/nxt_utf8_file_name_test.c +++ /dev/null @@ -1,148 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -extern char **environ; - -static nxt_int_t nxt_utf8_file_name_test(nxt_thread_t *thr); - - -nxt_module_init_t nxt_init_modules[1]; -nxt_uint_t nxt_init_modules_n; - - -int nxt_cdecl -main(int argc, char **argv) -{ - nxt_thread_t *thr; - - if (nxt_lib_start("utf8_file_name_test", argv, &environ) != NXT_OK) { - return 1; - } - - nxt_main_log.level = NXT_LOG_INFO; - - thr = nxt_thread(); - - if (nxt_utf8_file_name_test(thr) != NXT_OK) { - return 1; - } - - return 0; -} - - -static nxt_int_t -nxt_utf8_file_name_test(nxt_thread_t *thr) -{ - u_char *p, test[4], buf[32]; - ssize_t n; - uint32_t uc, lc; - nxt_int_t ret; - nxt_task_t task; - nxt_file_t uc_file, lc_file; - const u_char *pp; - nxt_file_name_t uc_name[10], lc_name[10]; - static const u_char utf8[4] NXT_NONSTRING = "UTF8"; - - nxt_thread_time_update(thr); - - uc_name[0] = 'u'; - uc_name[1] = 't'; - uc_name[2] = 'f'; - uc_name[3] = '8'; - uc_name[4] = '_'; - - lc_name[0] = 'u'; - lc_name[1] = 't'; - lc_name[2] = 'f'; - lc_name[3] = '8'; - lc_name[4] = '_'; - - nxt_memzero(&uc_file, sizeof(nxt_file_t)); - - uc_file.name = uc_name; - uc_file.log_level = NXT_LOG_ALERT; - - nxt_memzero(&lc_file, sizeof(nxt_file_t)); - - lc_file.name = lc_name; - - task.thread = thr; - task.log = thr->log; - - for (uc = 0x41; uc < 0x110000; uc++) { - - p = nxt_utf8_encode(&uc_name[5], uc); - - if (p == NULL) { - nxt_log_alert(thr->log, "nxt_utf8_encode(%05uxD) failed", uc); - return NXT_ERROR; - } - - *p = '\0'; - - pp = &uc_name[5]; - lc = nxt_utf8_lowcase(&pp, p); - - if (lc == 0xFFFFFFFF) { - nxt_log_alert(thr->log, "nxt_utf8_lowcase(%05uxD) failed: %05uxD", - uc, lc); - return NXT_ERROR; - } - - if (uc == lc) { - continue; - } - - p = nxt_utf8_encode(&lc_name[5], lc); - - if (p == NULL) { - nxt_log_alert(thr->log, "nxt_utf8_encode(%05uxD) failed", lc); - return NXT_ERROR; - } - - *p = '\0'; - - ret = nxt_file_open(&task, &uc_file, NXT_FILE_WRONLY, NXT_FILE_TRUNCATE, - NXT_FILE_DEFAULT_ACCESS); - if (ret != NXT_OK) { - return NXT_ERROR; - } - - if (nxt_file_write(&uc_file, utf8, 4, 0) != 4) { - return NXT_ERROR; - } - - nxt_file_close(&task, &uc_file); - - ret = nxt_file_open(&task, &lc_file, NXT_FILE_RDONLY, NXT_FILE_OPEN, - NXT_FILE_DEFAULT_ACCESS); - - if (ret == NXT_OK) { - n = nxt_file_read(&lc_file, test, 4, 0); - - nxt_file_close(&task, &lc_file); - - if (n != 4 || memcmp(utf8, test, 4) != 0) { - nxt_log_alert(thr->log, "nxt_file_read() mismatch"); - - nxt_file_delete(lc_file.name); - } - - p = nxt_sprintf(buf, buf + 32, "%04uXD; C; %04uXD;%n", uc, lc); - - nxt_fd_write(nxt_stdout, buf, p - buf); - } - - nxt_file_delete(uc_file.name); - } - - nxt_log_error(NXT_LOG_NOTICE, thr->log, "utf8 file name test passed"); - return NXT_OK; -} diff --git a/src/test/nxt_utf8_test.c b/src/test/nxt_utf8_test.c index fa36dc225..bdfa2bc9b 100644 --- a/src/test/nxt_utf8_test.c +++ b/src/test/nxt_utf8_test.c @@ -83,7 +83,6 @@ nxt_utf8_test(nxt_thread_t *thr) { u_char *p, utf8[4]; size_t len; - int32_t n; uint32_t u, d; nxt_uint_t i, k, l, m; const u_char *pp; @@ -194,16 +193,6 @@ nxt_utf8_test(nxt_thread_t *thr) } } - n = nxt_utf8_casecmp((u_char *) "ABC АБВ ΑΒΓ", - (u_char *) "abc абв αβγ", - nxt_length("ABC АБВ ΑΒΓ"), - nxt_length("abc абв αβγ")); - - if (n != 0) { - nxt_log_alert(thr->log, "nxt_utf8_casecmp() failed"); - return NXT_ERROR; - } - nxt_log_error(NXT_LOG_NOTICE, thr->log, "utf8 test passed"); return NXT_OK; } From 4b6a77e6e625f7696820e2cd4e262ca7aabd0269 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 12 Sep 2026 20:51:15 +0200 Subject: [PATCH 127/391] java: take the server name from the router, not the Host field getServerName() read the raw "Host" field and cut it at the first colon. For "Host: [::1]:8080" that colon is inside the IPv6 literal, so the servlet saw "[". Every other module reads server_name from the request, which the router has already normalised in nxt_http_validate_host(): the port is removed with the literal understood, a trailing dot is stripped, the name is lowercased, and an absent Host becomes "localhost". Java was the only one still parsing the field itself, and it got all four of those wrong, not just the literal. The fallback to getLocalName() goes with it. The router writes server_name on every request and gives it "localhost" when there is no Host, so the field is never missing -- and falling back to the local address would have been the worse answer anyway. getServerPort() already reads local_port and does not change. Co-Authored-By: Claude Opus 5 (1M context) --- src/java/nxt_jni_Request.c | 19 ++----------------- test/java/server_name/app.java | 20 ++++++++++++++++++++ test/test_java_application.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 17 deletions(-) create mode 100644 test/java/server_name/app.java diff --git a/src/java/nxt_jni_Request.c b/src/java/nxt_jni_Request.c index 4c3ce534e..1e31f57b0 100644 --- a/src/java/nxt_jni_Request.c +++ b/src/java/nxt_jni_Request.c @@ -597,27 +597,12 @@ nxt_java_Request_getScheme(JNIEnv *env, jclass cls, jlong req_ptr) static jstring JNICALL nxt_java_Request_getServerName(JNIEnv *env, jclass cls, jlong req_ptr) { - char *host, *colon; - nxt_unit_field_t *f; nxt_unit_request_t *r; r = nxt_jlong2ptr(req_ptr); - f = nxt_java_findHeader(r->fields, r->fields + r->fields_count, - "Host", 4); - if (f != NULL) { - host = nxt_unit_sptr_get(&f->value); - - colon = memchr(host, ':', f->value_length); - - if (colon == NULL) { - colon = host + f->value_length; - } - - return nxt_java_newString(env, host, colon - host); - } - - return nxt_java_Request_getLocalName(env, cls, req_ptr); + return nxt_java_newString(env, nxt_unit_sptr_get(&r->server_name), + r->server_name_length); } diff --git a/test/java/server_name/app.java b/test/java/server_name/app.java new file mode 100644 index 000000000..1ce818a15 --- /dev/null +++ b/test/java/server_name/app.java @@ -0,0 +1,20 @@ +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@WebServlet("/") +public class app extends HttpServlet +{ + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException + { + response.addHeader("X-Server-Name", request.getServerName()); + response.addHeader("X-Server-Port", + String.valueOf(request.getServerPort())); + } +} diff --git a/test/test_java_application.py b/test/test_java_application.py index 28d692af6..fe457fbdd 100644 --- a/test/test_java_application.py +++ b/test/test_java_application.py @@ -838,6 +838,38 @@ def test_java_application_no_method(): assert client.post()['status'] == 405, 'no method' +def test_java_application_server_name(): + client.load('server_name') + + def headers(host): + return client.get(headers={'Host': host, 'Connection': 'close'})[ + 'headers' + ] + + def server_name(host): + return headers(host)['X-Server-Name'] + + assert server_name('localhost') == 'localhost', 'plain host' + assert server_name('localhost:8080') == 'localhost', 'port stripped' + assert server_name('LocalHost') == 'localhost', 'lowercased' + assert server_name('localhost.') == 'localhost', 'trailing dot stripped' + assert server_name('[::1]') == '[::1]', 'ipv6 literal' + assert server_name('[::1]:8080') == '[::1]', 'ipv6 literal with port' + + # the port comes from the listener, never from the Host field + assert headers('[::1]:9999')['X-Server-Port'] == '8080', 'server port' + + # HTTP/1.0 may omit Host. The router substitutes "localhost" rather than + # leaving the name empty, so that is what the servlet sees -- previously + # this module fell back to the listener address instead. Aligning with + # every other module is the point of the change, not a side effect. + resp = client.http( + b'GET / HTTP/1.0\r\nConnection: close\r\n\r\n', raw=True + ) + assert resp['status'] == 200, 'no host' + assert resp['headers']['X-Server-Name'] == 'localhost', 'no host name' + + def test_java_application_get_header(): client.load('get_header') From 54a6a49061f48a3f3834cb041420543d4a422e1e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 12 Sep 2026 19:26:57 +0200 Subject: [PATCH 128/391] wasm: bound how long a guest may run A component yields only at an async wasi import, so one that computes without calling any stays inside a single call_handle() poll for as long as it likes. JoinHandle::abort() is never observed, the store is held, and a runtime worker thread is pinned. Add "execution_timeout" to the wasm-wasi-component object, in seconds, 0 (the default) meaning unbounded -- the same opt-in shape as limits.start_timeout. It is not under "limits" because that object is shared by every application type, so a wasm-only key there would validate for python or php and then be silently ignored. This is not a request timeout and does not replace limits.timeout, which bounds what the client waits for and answers 503. This bounds the guest: it runs from the guest's first instruction until handle() returns, and a guest parked in a host call keeps spending it. Epoch interruption rather than fuel: fuel counts every instruction to make a trap reproducible, which a deadline does not need, and costs 2-3x more. Epoch checks a counter at function entries and loop headers -- entries as well, because a call tree with no back-edge would otherwise run unchecked. The engine only enables it when the key is set, so an application that does not ask for a deadline generates the same code as before. The epoch is advanced by its own thread rather than a task on the module's runtime: a guest that never yields occupies a runtime worker, so enough of them would starve the very task meant to preempt them. The thread waits on a condvar while nothing is in flight, and a guard taken when the deadline is armed releases it when the task finishes or is aborted, so an idle worker costs no wakeups. The deadline is armed inside the guest task rather than at store creation: set_epoch_deadline() counts from the epoch at the moment it is called, so arming it earlier billed the queue wait and the blocking body read to the guest. A deadline of n ticks stops the guest somewhere in [(n-1)*TICK, n*TICK] because the store is created between ticks, so the tick count is rounded up and one added: a request is never cut short of its timeout. A whole number of seconds is a whole number of ticks, so the overshoot is one tick, plus however long the guest takes to reach its next epoch check. The trap surfaces as an ordinary Err from call_handle(), which the existing wrapper turns into a 500 when nothing has gone out yet, and into a truncated response once the guest has published its headers -- no new panic path. Co-Authored-By: Claude Opus 5 (1M context) --- docs/unit-openapi.yaml | 11 ++ src/nxt_application.h | 16 ++ src/nxt_conf_validation.c | 40 ++++ src/nxt_main_process.c | 5 + src/wasm-wasi-component/src/lib.rs | 171 +++++++++++++++++- test/test_wasm-wasi-component.py | 154 ++++++++++++++++ test/unit/applications/lang/wasm_component.py | 22 ++- test/wasm_component/spin_loop/Cargo.lock | 34 ++++ test/wasm_component/spin_loop/Cargo.toml | 18 ++ test/wasm_component/spin_loop/src/lib.rs | 22 +++ test/wasm_component/spin_loop/wit/world.wit | 6 + 11 files changed, 490 insertions(+), 9 deletions(-) create mode 100644 test/wasm_component/spin_loop/Cargo.lock create mode 100644 test/wasm_component/spin_loop/Cargo.toml create mode 100644 test/wasm_component/spin_loop/src/lib.rs create mode 100644 test/wasm_component/spin_loop/wit/world.wit diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index a81d78778..065b51e39 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -7136,6 +7136,17 @@ components: $ref: "#/components/schemas/stringArray" description: "Host directories this application may have access to." + execution_timeout: + type: integer + description: "Time in seconds a component may run for one request + before Unit stops it; 0, the default, means no limit. Unit + answers 500 if the component has not begun its response; once + its headers are sent, the client sees a truncated response + instead. This bounds the guest, not the request: it runs from + the guest's first instruction until it returns, waits inside + host calls included. To bound what the client waits for, use + limits/timeout, which answers 503." + configApplicationPHP: description: "PHP application on Unit." allOf: diff --git a/src/nxt_application.h b/src/nxt_application.h index e9c74afa2..0b78ea971 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -110,6 +110,22 @@ typedef struct { const char *component; nxt_conf_value_t *access; + + /* + * "execution_timeout" in milliseconds, 0 for unbounded. It bounds one + * invocation of the component's "handle", and only the wasm module acts + * on it: the guest is preempted inside the module's own process, so + * unlike "limits": {"start_timeout"} the router has nothing to arm. + * + * It is wall clock from the guest's first instruction until "handle" + * returns, not CPU time, so time the guest spends parked in a host + * call -- waiting for a slow client to drain the response body, say -- + * is spent too. And the deadline is only checked while wasm executes, + * so a guest parked in a host call is not preempted, only trapped when + * it resumes. This is not a request timeout; that is + * "limits": {"timeout"}, which the router arms for every app type. + */ + nxt_msec_t execution_timeout; } nxt_wasm_wc_app_conf_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 7c8ce63b8..dae8e3854 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -209,6 +209,8 @@ static nxt_int_t nxt_conf_vldt_environment(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value); static nxt_int_t nxt_conf_vldt_c_string(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_wasm_wc_timeout(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_targets_exclusive( nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_targets(nxt_conf_validation_t *vldt, @@ -1290,6 +1292,10 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_wasm_wc_members[] = { .type = NXT_CONF_VLDT_OBJECT, .validator = nxt_conf_vldt_object, .u.members = nxt_conf_vldt_wasm_access_members, + }, { + .name = nxt_string("execution_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_wasm_wc_timeout, }, NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) @@ -4075,6 +4081,40 @@ nxt_conf_vldt_environment(nxt_conf_validation_t *vldt, nxt_str_t *name, } +/* + * "execution_timeout" bounds one invocation of the component's "handle". + * It reaches the module through NXT_CONF_MAP_MSEC (nxt_wasm_wc_app_conf[], + * src/nxt_main_process.c), which computes (nxt_msec_t) seconds * 1000 and + * range-checks nothing, so bound it here exactly as "start_timeout" is + * bounded in nxt_conf_vldt_app_limits() above: a negative value is an + * out-of-range conversion to an unsigned type, and seconds above + * NXT_INT32_T_MAX / 1000 land past the sign bit of the 32-bit millisecond + * value. 0 means unbounded. + */ + +static nxt_int_t +nxt_conf_vldt_wasm_wc_timeout(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + int64_t timeout; + + timeout = nxt_conf_get_number(value); + + if (timeout < 0) { + return nxt_conf_vldt_error(vldt, "The \"execution_timeout\" number " + "must not be negative."); + } + + if (timeout > NXT_INT32_T_MAX / 1000) { + return nxt_conf_vldt_error(vldt, "The \"execution_timeout\" number " + "must not exceed %d.", + NXT_INT32_T_MAX / 1000); + } + + return NXT_OK; +} + + /* * Reject empty and embedded-NUL values for options that are later consumed * as NUL-terminated C strings (NXT_CONF_MAP_CSTRZ or a direct sink): diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 52fb4ebe9..e6d666255 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -432,6 +432,11 @@ static nxt_conf_map_t nxt_wasm_wc_app_conf[] = { NXT_CONF_MAP_PTR, offsetof(nxt_common_app_conf_t, u.wasm_wc.access), }, + { + nxt_string("execution_timeout"), + NXT_CONF_MAP_MSEC, + offsetof(nxt_common_app_conf_t, u.wasm_wc.execution_timeout), + }, }; diff --git a/src/wasm-wasi-component/src/lib.rs b/src/wasm-wasi-component/src/lib.rs index f817486d5..6c4bce706 100644 --- a/src/wasm-wasi-component/src/lib.rs +++ b/src/wasm-wasi-component/src/lib.rs @@ -9,7 +9,8 @@ use std::mem::MaybeUninit; use std::os::raw::c_int; use std::process::exit; use std::ptr; -use std::sync::OnceLock; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock}; +use std::time::Duration; use tokio::sync::mpsc; use wasmtime::component::{Component, Linker, ResourceTable}; use wasmtime::{Config, Engine, Store}; @@ -102,6 +103,7 @@ unsafe extern "C" fn setup( let result = GLOBAL_CONFIG.set(GlobalConfig { component: component.to_string(), dirs, + execution_timeout: wasm_conf.execution_timeout.into(), }); assert!(result.is_ok()); Ok(()) @@ -192,6 +194,8 @@ unsafe extern "C" fn request_handler( struct GlobalConfig { component: String, dirs: Vec, + /// "execution_timeout" in milliseconds, 0 for unbounded. + execution_timeout: u64, } struct GlobalState { @@ -199,6 +203,32 @@ struct GlobalState { component: ProxyPre, global_config: &'static GlobalConfig, sender: mpsc::Sender, + /// `None` unless "execution_timeout" is configured, in which case the + /// engine has epoch interruption on and every store needs a deadline. + epoch: Option, +} + +/// What a store needs to be armed: who advances the epoch, and how far +/// ahead the deadline sits. +/// +/// The two travel together because neither is correct alone. With +/// `epoch_interruption(true)` a store that is never given a deadline traps +/// on its first instruction, and a deadline nothing advances never fires, +/// so a single `Option` is what keeps a later `Store::new` from getting +/// either. +struct Epoch { + ticker: Arc, + /// The deadline, in epoch ticks. See `epoch_deadline_ticks()`. + deadline: u64, +} + +impl Epoch { + /// Arms `store` and keeps the ticker awake until the guard is dropped. + fn arm(&self, store: &mut Store) -> Ticking { + store.set_epoch_deadline(self.deadline); + + self.ticker.ticking() + } } impl GlobalState { @@ -214,8 +244,36 @@ impl GlobalState { // * Memory limits/etc. let mut config = Config::new(); config.wasm_component_model(true); + + // A guest yields only at an async wasi import, so one that computes + // without calling any stays inside a single `call_handle()` poll for + // as long as it likes: `JoinHandle::abort()` is never observed, the + // store is held, and a runtime worker thread is pinned. Epoch + // interruption is what lets the host stop it. Fuel would too, but + // it counts every instruction to be deterministic, where this only + // reads a counter at function entries and loop headers and costs + // 2-3x less (wasmtime Config::epoch_interruption docs); a deadline + // does not need to be reproducible, only enforced. + // + // Off unless asked for: it changes the code generated for every + // guest, and a component that legitimately computes for a long time + // must keep working. + let bounded = global_config.execution_timeout != 0; + + if bounded { + config.epoch_interruption(true); + } + let engine = Engine::new(&config)?; + // One `bounded` for both: with epoch interruption on, a store that + // is never armed traps at once, so the engine setting and the + // `Epoch` that arms every store cannot be allowed to disagree. + let epoch = bounded.then(|| Epoch { + ticker: EpochTicker::start(engine.clone()), + deadline: epoch_deadline_ticks(global_config.execution_timeout), + }); + // Compile the binary component on disk in Wasmtime. This is then // pre-instantiated with host APIs defined by WASI. The result of // this is a "pre-instantiated instance" which can be used to @@ -250,6 +308,7 @@ impl GlobalState { component: proxy, sender, global_config, + epoch, }) } @@ -339,6 +398,17 @@ impl GlobalState { // generate headers, write those below, and then compute the body // afterwards. let mut task = GuestTask(Some(tokio::spawn(async move { + // Arm the deadline here rather than beside `Store::new`, and + // hold the ticker awake for as long as the guest can run. The + // budget is spent by the epoch, which advances in wall clock + // while any request is in flight, so everything before this + // point would spend it without the guest running: the wait for + // a runtime worker -- under load, behind the very guests this + // bounds -- and the blocking `request_read()` inside + // `to_request_body()`. A task that is dropped before it is + // ever polled now also never wakes the ticker. + let _ticking = self.epoch.as_ref().map(|e| e.arm(&mut store)); + let req = store .data_mut() .http() @@ -484,6 +554,105 @@ impl GlobalState { } } +/// How often the epoch advances while a guest is running. +const EPOCH_TICK: Duration = Duration::from_millis(100); + +/// Turn a timeout in milliseconds into `Store::set_epoch_deadline()` ticks. +/// +/// A store is armed at an arbitrary point between two ticks, so with `n` +/// ticks the guest is stopped somewhere in `[(n - 1) * TICK, n * TICK]`. +/// The extra tick is what keeps the lower end at or after the configured +/// timeout: a request is never cut short of it. +/// +/// The far end is one tick past the timeout once the rounding up is a +/// no-op, which it always is here -- the config is whole seconds, so +/// `timeout_ms` is a multiple of `TICK` -- and two ticks otherwise. Add +/// to that whatever the guest takes to reach its next epoch check, which +/// wasmtime puts at function entries and loop headers. +fn epoch_deadline_ticks(timeout_ms: u64) -> u64 { + let tick = EPOCH_TICK.as_millis() as u64; + + timeout_ms.saturating_add(tick - 1) / tick + 1 +} + +/// Advances the engine's epoch so an armed deadline can fire. +/// +/// The ticker is its own OS thread rather than a task on the module's tokio +/// runtime. A guest that never yields occupies a runtime worker thread for +/// as long as it runs, so enough of them starve exactly the task that is +/// meant to preempt them -- on a one-core machine, the first one does. A +/// thread outside the runtime cannot be starved by the code it bounds. +/// +/// It also sleeps while nothing is running. A worker with no request in +/// flight has nothing to preempt, and a server can hold many idle workers, +/// so the ticker waits on `wake` until a request arms a deadline. +struct EpochTicker { + running: Mutex, + wake: Condvar, +} + +impl EpochTicker { + fn start(engine: Engine) -> Arc { + let ticker = Arc::new(EpochTicker { + running: Mutex::new(0), + wake: Condvar::new(), + }); + + let thread = ticker.clone(); + std::thread::spawn(move || thread.run(engine)); + + ticker + } + + fn run(&self, engine: Engine) { + loop { + { + let mut running = self.lock(); + + while *running == 0 { + running = match self.wake.wait(running) { + Ok(running) => running, + Err(e) => e.into_inner(), + }; + } + } + + // Sleeping outside the lock means a request that finishes + // meanwhile still gets this increment. That is one wasted + // atomic add, not a missed deadline. + std::thread::sleep(EPOCH_TICK); + engine.increment_epoch(); + } + } + + /// Keeps the ticker awake until the returned guard is dropped. + fn ticking(self: &Arc) -> Ticking { + *self.lock() += 1; + self.wake.notify_one(); + + Ticking(self.clone()) + } + + /// `panic = 'abort'` means a lock is never poisoned, but taking the + /// inner value rather than unwrapping keeps a second panic -- which + /// would end the worker -- off the table if that ever changes. + fn lock(&self) -> MutexGuard<'_, usize> { + match self.running.lock() { + Ok(running) => running, + Err(e) => e.into_inner(), + } + } +} + +/// Marks one request as running, so the ticker keeps the epoch moving. +struct Ticking(Arc); + +impl Drop for Ticking { + fn drop(&mut self) { + *self.0.lock() -= 1; + } +} + /// Owns the wasm task so that leaving `handle_inner()` early stops it. /// Dropping a `JoinHandle` only detaches the task, it does not cancel it: /// a client that disconnects mid-response fails the write, the request is diff --git a/test/test_wasm-wasi-component.py b/test/test_wasm-wasi-component.py index 30a244f85..54e13930e 100644 --- a/test/test_wasm-wasi-component.py +++ b/test/test_wasm-wasi-component.py @@ -1,5 +1,9 @@ +import subprocess +import time + import pytest from unit.applications.lang.wasm_component import ApplicationWasmComponent +from unit.log import Log prerequisites = { 'modules': {'wasm-wasi-component': 'any'}, @@ -41,6 +45,32 @@ def conf_component(value): assert 'must not be empty' in resp.get('detail', ''), 'empty' +def test_wasm_component_execution_timeout_invalid(): + # "execution_timeout" is seconds and reaches the module as a 32-bit + # millisecond value, so both ends are bounded at validation. + def conf_timeout(value): + return client.conf( + { + "app": { + "type": "wasm-wasi-component", + "processes": {"spare": 0}, + "component": "/app.wasm", + "execution_timeout": value, + } + }, + 'applications', + ) + + resp = conf_timeout(-1) + assert 'must not be negative' in resp.get('detail', ''), 'negative' + + resp = conf_timeout(2147483 + 1) + assert 'must not exceed' in resp.get('detail', ''), 'too large' + + assert 'success' in conf_timeout(2147483), 'largest' + assert 'success' in conf_timeout(0), 'unbounded' + + def test_wasm_component_obs_text(): client.load('hello_world') @@ -95,3 +125,127 @@ def test_wasm_component_request_body(): resp = client.post(body='x' * size) assert resp['status'] == 200, size assert resp['body'] == 'Hello', size + + +def test_wasm_component_execution_timeout(): + # "spin_loop" never yields and never answers. A guest only yields at an + # async wasi import, so without a deadline it stays inside a single + # call_handle() poll forever, holding its store and a runtime thread. + # "execution_timeout" arms the wasmtime epoch deadline that stops it. + client.load('spin_loop', execution_timeout=1) + + assert client.get()['status'] == 500, 'preempted' + + # The worker survived the trap and serves the next request, which the + # deadline stops the same way. A trap is an ordinary Err out of + # call_handle(), not a panic -- under `panic = 'abort'` a panic would + # take the worker with it. + # + # Time this one, not the first: "spare" is 0, so the first request pays + # for starting the process and compiling the component, which would let + # a deadline that fired at 100 ms still clear the lower bound. + start = time.time() + assert client.get()['status'] == 500, 'worker alive' + elapsed = time.time() - start + + # Not cut short of the configured timeout, and not far past it. The + # deadline is 1 s plus at most two 100 ms ticks; the rest of the margin + # is the request itself. + assert elapsed >= 1, elapsed + assert elapsed < 5, elapsed + + # Assert the mechanism, not just the status: a guest that trapped for its + # own reasons would answer 500 through the same path. The epoch deadline + # renders as an "interrupt" trap. + assert Log.findall(r'wasm trap: interrupt') != [], 'stopped by the deadline' + + assert Log.findall(r'signal 6') == [], 'no abort' + assert Log.findall(r'panicked at') == [], 'no panic' + + +def test_wasm_component_execution_timeout_body(skip_fds_check): + # A request with a body still gets its full deadline. The deadline is + # armed inside the guest task rather than beside Store::new(), so the + # body read no longer comes out of the guest's budget. + # + # This pins the property, not the regression: the smallest configurable + # timeout is one second and the router hands the module a body that is + # already complete, so reading a few megabytes costs milliseconds and + # this passes under the old arming too. Catching that directly would + # need a body large enough to take about a second to read. + skip_fds_check(router=True) + + client.load('spin_loop', execution_timeout=1) + + start = time.time() + assert client.post(body='x' * (4 * 1024 * 1024))['status'] == 500, 'preempted' + elapsed = time.time() - start + + # The guest still gets its full second after the body has been read. + assert elapsed >= 1, elapsed + + assert Log.findall(r'signal 6') == [], 'no abort' + + + +def _reap_spinning(app): + """Kill a wasm worker that will not stop on its own. + + Without a deadline the guest never yields, so nothing in Unit can reclaim + the worker: the router answers the client and unlinks the rpc data, but + the process keeps running. Reap it here so it does not outlive the test + -- the same shape as test_app_start_timeout.py's _reap_stuck(). + """ + + out = subprocess.run( + ['ps', '-eo', 'pid,args'], capture_output=True, text=True, check=False + ).stdout + + for line in out.splitlines(): + if f'unit: "{app}" application' in line: + subprocess.run( + ['kill', '-9', line.split()[0]], check=False + ) + + +def test_wasm_component_execution_timeout_absent(skip_alert, skip_fds_check): + # The control for test_wasm_component_execution_timeout: the same guest + # with no deadline is never stopped. "limits": {"timeout"} bounds what + # the CLIENT waits for -- the router answers 503 and unlinks the request + # -- but it does not reach the application, so the worker spins on and + # this test reaps it. + # + # That difference is the point: 503-from-the-router is not + # 500-from-the-deadline, and "execution_timeout" is not a request timeout. + skip_alert(r'exited on signal') + skip_fds_check(router=True) + + try: + client.load('spin_loop', limits={"timeout": 3}) + + start = time.time() + assert client.get(read_timeout=30)['status'] == 503, 'router gave up' + elapsed = time.time() - start + + assert elapsed >= 3, elapsed + + assert Log.findall(r'wasm trap: interrupt') == [], 'no deadline fired' + + finally: + _reap_spinning('spin_loop') + + +def test_wasm_component_execution_timeout_unreached(): + # A deadline a normal guest never approaches stays out of the way. The + # extra tick in the tick computation is there so a request is never cut + # short of its timeout; this is the other half of that -- arming the + # deadline must not cost a request that answers immediately. + client.load('hello_world', execution_timeout=1) + + resp = client.get() + + assert resp['status'] == 200, 'answered' + assert resp['body'] == 'Hello', 'body' + + # And again on a warm worker, where the store is armed per request. + assert client.get()['status'] == 200, 'warm' diff --git a/test/unit/applications/lang/wasm_component.py b/test/unit/applications/lang/wasm_component.py index a6c8dd140..f0eb5b1ad 100644 --- a/test/unit/applications/lang/wasm_component.py +++ b/test/unit/applications/lang/wasm_component.py @@ -41,23 +41,29 @@ def prepare_env(script): return output - def load(self, script, **kwargs): + def load(self, script, execution_timeout=None, limits=None, **kwargs): self.prepare_env(script) component_path = f'{option.temp_dir}/wasm_component/{script}/target/wasm32-wasip1/release/test_wasi_component.wasm' + app = { + "type": "wasm-wasi-component", + "processes": {"spare": 0}, + "component": component_path, + } + + if execution_timeout is not None: + app["execution_timeout"] = execution_timeout + + if limits is not None: + app["limits"] = limits + self._load_conf( { "listeners": { "*:8080": {"pass": f"applications/{quote(script, '')}"} }, - "applications": { - script: { - "type": "wasm-wasi-component", - "processes": {"spare": 0}, - "component": component_path, - } - }, + "applications": {script: app}, }, **kwargs, ) diff --git a/test/wasm_component/spin_loop/Cargo.lock b/test/wasm_component/spin_loop/Cargo.lock new file mode 100644 index 000000000..2daeb73de --- /dev/null +++ b/test/wasm_component/spin_loop/Cargo.lock @@ -0,0 +1,34 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "test-wasi-component" +version = "0.1.0" +dependencies = [ + "bitflags", + "wasi", + "wit-bindgen-rt", +] + +[[package]] +name = "wasi" +version = "0.13.0+wasi-0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652cd73449d0b957a2743b70c72d79d34a5fa505696488f4ca90b46f6da94118" +dependencies = [ + "bitflags", + "wit-bindgen-rt", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "026d24a27f6712541fa534f2954bd9e0eb66172f033c2157c0f31d106255c497" diff --git a/test/wasm_component/spin_loop/Cargo.toml b/test/wasm_component/spin_loop/Cargo.toml new file mode 100644 index 000000000..a87fbeb5a --- /dev/null +++ b/test/wasm_component/spin_loop/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "test-wasi-component" +version = "0.1.0" +edition = "2021" + +[dependencies] +bitflags = "2.4.2" +wit-bindgen-rt = "0.21.0" +wasi = "0.13.0" + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "component:test-wasi-component" +proxy = true + +[package.metadata.component.dependencies] diff --git a/test/wasm_component/spin_loop/src/lib.rs b/test/wasm_component/spin_loop/src/lib.rs new file mode 100644 index 000000000..66210332b --- /dev/null +++ b/test/wasm_component/spin_loop/src/lib.rs @@ -0,0 +1,22 @@ +use wasi::http::types::{IncomingRequest, ResponseOutparam}; + +wasi::http::proxy::export!(Component); + +struct Component; + +// A guest that never yields and never answers. wasi-http is linked async, +// but a component only yields at an async wasi import, so this loop keeps +// the host inside a single call_handle() poll for as long as it runs. It +// exists to prove the host can preempt such a guest; without a deadline the +// request never completes and the worker thread stays pinned. +// +// black_box() keeps the loop from being optimised away. +impl wasi::exports::http::incoming_handler::Guest for Component { + fn handle(_request: IncomingRequest, _response_out: ResponseOutparam) { + let mut n: u64 = 0; + + loop { + n = core::hint::black_box(n).wrapping_add(1); + } + } +} diff --git a/test/wasm_component/spin_loop/wit/world.wit b/test/wasm_component/spin_loop/wit/world.wit new file mode 100644 index 000000000..82c810ef1 --- /dev/null +++ b/test/wasm_component/spin_loop/wit/world.wit @@ -0,0 +1,6 @@ +package component:test-wasi-component; + +/// An example world for the component to target. +world example { + export hello-world: func() -> string; +} From fd2d15013b9b2668845382c88e0afb61a7d3769f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 12 Sep 2026 20:43:21 +0200 Subject: [PATCH 129/391] static: send Last-Modified in GMT, not local time nxt_http_static.c converted the file mtime with nxt_localtime() and then handed the result to nxt_http_date(), which appends the literal " GMT". On a host not set to UTC the numbers were local and the label was wrong: a CEST box emitted "Sun, 13 Sep 2020 14:26:40 GMT" for mtime 1600000000, two hours ahead of the real GMT value. RFC 9110 Sect. 5.6.7 requires an HTTP-date to be GMT, and a cache takes the label at face value. Unit does not evaluate conditional requests, so the effect is entirely downstream, and it is the RFC 9111 Sect. 4.2.2 heuristic freshness lifetime, a fraction of "Date - Last-Modified", that carries it: - West of UTC the value is in the past, so the interval is too large and a cache holds the object longer than it should: stale content. - East of UTC the value is in the future, the interval is <= 0 and the lifetime collapses to zero, so a cache revalidates every time: needless 200s. RFC 9110 Sect. 8.8.2.1 also forbids an origin from sending a Last-Modified later than its own Date. Steady-state If-Modified-Since is consistent in both directions, since the client echoes back the same skewed value the cache stored. Use nxt_gmtime(), which is what the Date response header already goes through (nxt_http_date_cache is NXT_THREAD_TIME_GMT). The access log's $time_local stays local, which its name promises. nxt_gmtime() was documented for positive times only, and a file mtime can be negative: it derived the time of day from an unsigned "s % 86400" and rendered mtime -1 as "Thu, 01 Jan 1970 1193046:28:1", wide enough to push the " GMT" out of the buffer and emit a malformed header. Only the two extraction steps assumed a positive input -- the Gauss' formula body counts from March 1, 1 BCE, so it is already correct for any date after year 1 -- so floor the division instead of truncating it and take the weekday in signed arithmetic. Every other caller passes the current time, so nothing that worked before changes. On QNX, where nxt_time_t is a signed int32_t over an unsigned native time_t, an mtime past 2038 arrives negative and now renders as a date in 1901 rather than as the old local-time rendering. That is the Y2038 limitation nxt_types.h names in its own comment; it already costs the Date header and this handler's ETag, and it cannot be repaired at the call site, since nxt_gmtime() takes an nxt_time_t. Filed as #329. Every date the type can represent is more correct than before. Upgrade note: east of UTC, for up to one UTC offset after this lands, an If-Modified-Since a client captured from the old skewed header can still satisfy a caching proxy's own 304 check for a file modified inside that window, so such a client keeps the old copy a little longer. The pytest cases pin Last-Modified to the exact GMT rendering of a known mtime and of a pre-epoch one -- the exact string, because parsedate_to_datetime() discards the day name and a wrong weekday would otherwise parse to the right instant. The first can only fail where localtime() and gmtime() differ, so it skips itself, loudly, under a zero offset, and the pytest legs now run under TZ=Asia/Kolkata: never UTC, no DST, and a half-hour offset that catches whole-hour assumptions. GitHub runners are UTC, and glibc reads an unknown zone as UTC too, so without both halves a runner without tzdata would go green against a broken build. nxt_gmtime_test now covers pre-epoch times, and a second loop walks a day of seconds across the epoch -- the existing loop steps whole days, so it never checked a time of day at all. The pre-epoch cases are fixed calendar fields rather than a comparison against gmtime(): nxt_time_t and time_t need not agree on a negative value, on QNX in signedness and elsewhere in width, so only fixed expectations test the arithmetic everywhere. They were derived with "date -u". Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-test.yml | 9 ++++ src/nxt_gmtime.c | 33 ++++++++++-- src/nxt_http_static.c | 10 +++- src/test/nxt_gmtime_test.c | 93 +++++++++++++++++++++++++++++++- test/test_static.py | 56 +++++++++++++++++++ 5 files changed, 195 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 52ab2b969..b5b3965f1 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -530,6 +530,15 @@ jobs: if: steps.metadata.outputs.module != 'wasm' - name: Run ${{ steps.metadata.outputs.module }} tests + # GitHub runners are UTC, where localtime() and gmtime() agree, so a + # UTC-only run cannot tell a correct HTTP-date from one rendered in + # local time and labelled "GMT" (see test_static_last_modified_is_gmt, + # which skips itself, loudly, when the offset is zero). Kolkata is + # +05:30 all year: never UTC, no DST to shift it, and a half-hour + # offset that also catches whole-hour assumptions. + # "sudo -E" below keeps it for unitd, which inherits pytest's env. + env: + TZ: Asia/Kolkata run: | # Pin unitd's embedded interpreter to the toolcache prefix it was # built against; conftest turns this into PYTHONHOME for unitd only. diff --git a/src/nxt_gmtime.c b/src/nxt_gmtime.c index 9c3fa190c..01b6840f4 100644 --- a/src/nxt_gmtime.c +++ b/src/nxt_gmtime.c @@ -7,19 +7,44 @@ #include -/* The function is valid for positive nxt_time_t only. */ +/* + * The function is valid for any nxt_time_t that lands on a calendar date + * from year 1 onward -- that is the whole range of a 32-bit time_t, and + * every pre-epoch timestamp a filesystem can hold. + */ void nxt_gmtime(nxt_time_t s, struct tm *tm) { nxt_int_t yday; nxt_uint_t daytime, mday, mon, year, days, leap; + nxt_time_t dayno, secs; + + dayno = s / 86400; + secs = s % 86400; + + if (secs < 0) { + /* + * C integer division truncates toward zero, but a calendar floors: + * for a pre-epoch time the remainder comes out negative and the day + * number one too high. + */ + secs += 86400; + dayno--; + } - days = (nxt_uint_t) (s / 86400); - daytime = (nxt_uint_t) (s % 86400); + daytime = (nxt_uint_t) secs; /* January 1, 1970 was Thursday. */ - tm->tm_wday = (4 + days) % 7; + tm->tm_wday = (nxt_int_t) (((dayno + 4) % 7 + 7) % 7); + + /* + * A pre-epoch day number wraps when it is made unsigned, but every step + * below is modular addition and the result lands back in range, because + * the Gauss' formula counts from March 1, 1 BCE and so is positive for + * every date this function accepts. + */ + days = (nxt_uint_t) dayno; /* The algorithm based on Gauss' formula. */ diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index d0413659e..9a4c20b14 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -701,7 +701,15 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } - nxt_localtime(nxt_file_mtime(&fi), &tm); + /* + * On QNX nxt_time_t is a signed int32_t over an unsigned native + * time_t, so an mtime past 2038 arrives here negative and renders + * as a date in 1901. That is the Y2038 limitation nxt_types.h + * names in its own comment, it costs the Date header and the ETag + * below in the same way, and it cannot be repaired at this call + * site, since nxt_gmtime() takes an nxt_time_t. See #329. + */ + nxt_gmtime(nxt_file_mtime(&fi), &tm); field->value = p; field->value_length = nxt_http_date(p, &tm) - p; diff --git a/src/test/nxt_gmtime_test.c b/src/test/nxt_gmtime_test.c index cec81ab45..52fbf8e3c 100644 --- a/src/test/nxt_gmtime_test.c +++ b/src/test/nxt_gmtime_test.c @@ -22,20 +22,87 @@ #endif +/* + * Pre-epoch expectations, checked against fixed calendar fields rather than + * against gmtime(). Do not "simplify" these into the differential loops + * below: handing a nxt_time_t to gmtime(), which takes a time_t, is only + * safe where the two types agree, and for a negative value they need not. + * On QNX nxt_time_t is a signed int32_t while the native time_t is uint32_t, + * so the same bits are 1969 to nxt_gmtime() and 2106 to the C library. And + * where nxt_time_t is 8 bytes against a 4-byte time_t -- the case the + * NXT_TIME_T_SIZE test above already contemplates -- the library reads half + * the object: the low half on a little-endian machine, which often looks + * right by accident, and the high half on a big-endian one, which is + * garbage. Fixed fields test the arithmetic on every platform instead. + * + * A file's mtime can be negative ("touch -d 1969-07-20"), and the day-time + * step used to wrap and render an hour of 1193046. -432000 and beyond + * matter on their own: a weekday taken from an unsigned day number is still + * right for the first four days before the epoch and wrong after them. + * + * Rows derived with "date -u -d @" on a signed 64-bit time_t host, + * never from nxt_gmtime() itself. + */ + +static const struct { + nxt_time_t time; + int year, mon, mday, wday, yday, hour, min, sec; +} nxt_gmtime_pre_epoch[] = { + { -1, 69, 11, 31, 3, 364, 23, 59, 59 }, /* Wed, 31 Dec 1969 */ + { -86400, 69, 11, 31, 3, 364, 0, 0, 0 }, /* Wed, 31 Dec 1969 */ + { -432000, 69, 11, 27, 6, 360, 0, 0, 0 }, /* Sat, 27 Dec 1969 */ + { -432001, 69, 11, 26, 5, 359, 23, 59, 59 }, /* Fri, 26 Dec 1969 */ + { -14182940, 69, 6, 20, 0, 200, 20, 17, 40 }, /* Sun, 20 Jul 1969 */ + { -2147472000, 1, 11, 14, 6, 347, 0, 0, 0 }, /* Sat, 14 Dec 1901 */ +#if (NXT_TIME_T_SIZE == 8) + /* The oldest date the Gauss' formula supports. */ + { -62135596800, -1899, 0, 1, 1, 0, 0, 0, 0 }, /* Mon, 01 Jan 0001 */ +#endif +}; + + nxt_int_t nxt_gmtime_test(nxt_thread_t *thr) { + time_t native; struct tm tm0, *tm1; + nxt_uint_t i; nxt_time_t s; nxt_nsec_t start, end; nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "gmtime test started"); + for (i = 0; i < nxt_nitems(nxt_gmtime_pre_epoch); i++) { + + s = nxt_gmtime_pre_epoch[i].time; + + nxt_gmtime(s, &tm0); + + if (tm0.tm_year != nxt_gmtime_pre_epoch[i].year + || tm0.tm_mon != nxt_gmtime_pre_epoch[i].mon + || tm0.tm_mday != nxt_gmtime_pre_epoch[i].mday + || tm0.tm_wday != nxt_gmtime_pre_epoch[i].wday + || tm0.tm_yday != nxt_gmtime_pre_epoch[i].yday + || tm0.tm_hour != nxt_gmtime_pre_epoch[i].hour + || tm0.tm_min != nxt_gmtime_pre_epoch[i].min + || tm0.tm_sec != nxt_gmtime_pre_epoch[i].sec) + { + nxt_log_alert(thr->log, + "gmtime test failed: %T @ wday %d, " + "%02d.%02d.%d %02d:%02d:%02d", + s, tm0.tm_wday, tm0.tm_mday, tm0.tm_mon + 1, + tm0.tm_year + 1900, tm0.tm_hour, tm0.tm_min, + tm0.tm_sec); + return NXT_ERROR; + } + } + for (s = 0; s < NXT_GMTIME_MAX; s += 86400) { nxt_gmtime(s, &tm0); - tm1 = gmtime(&s); + native = (time_t) s; + tm1 = gmtime(&native); if (tm0.tm_mday != tm1->tm_mday || tm0.tm_mon != tm1->tm_mon @@ -51,6 +118,30 @@ nxt_gmtime_test(nxt_thread_t *thr) } } + /* + * The loop above steps whole days, so it never exercises the time of + * day. Walk a day's worth of seconds either side of the epoch. + */ + + for (s = 0; s < 90000; s++) { + + nxt_gmtime(s, &tm0); + native = (time_t) s; + tm1 = gmtime(&native); + + if (tm0.tm_hour != tm1->tm_hour + || tm0.tm_min != tm1->tm_min + || tm0.tm_sec != tm1->tm_sec + || tm0.tm_mday != tm1->tm_mday + || tm0.tm_wday != tm1->tm_wday) + { + nxt_log_alert(thr->log, + "gmtime test failed: %T @ %02d:%02d:%02d", + s, tm1->tm_hour, tm1->tm_min, tm1->tm_sec); + return NXT_ERROR; + } + } + nxt_thread_time_update(thr); start = nxt_thread_monotonic_time(thr); diff --git a/test/test_static.py b/test/test_static.py index f7fc0fa82..8eaf1cf47 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -1,5 +1,7 @@ import os import socket +import time +from email.utils import formatdate, parsedate_to_datetime from pathlib import Path import pytest @@ -170,6 +172,60 @@ def test_static_conditional_ignored(): assert resp['body'] == '0123456789', 'full body on future date' +def test_static_last_modified_is_gmt(temp_dir): + # RFC 9110 Sect. 5.6.7: an HTTP-date is GMT. Last-Modified used to be + # rendered from localtime() and then labelled "GMT", so on a host not set + # to UTC the value was wrong by the UTC offset while still looking valid. + # Only a downstream cache comparing it would ever notice. + mtime = 1600000000 # Sun, 13 Sep 2020 12:26:40 GMT + + # Under UTC the two renderings are identical and this proves nothing, so + # say so out loud rather than passing. glibc also treats an unknown zone + # as UTC, which is how a runner image without tzdata would go green + # against a broken build; the workflow pins a zone for this reason. + if time.localtime(mtime).tm_gmtoff == 0: + pytest.skip('TZ is UTC: localtime() and gmtime() cannot be told apart') + + os.utime(f'{temp_dir}/assets/index.html', (mtime, mtime)) + + last_modified = client.get(url='/index.html')['headers']['Last-Modified'] + + assert last_modified == formatdate( + mtime, usegmt=True + ), 'Last-Modified is the GMT rendering of mtime' + + +def test_static_last_modified_pre_epoch(temp_dir): + # A pre-epoch mtime is legal ("touch -d 1969-07-20"). nxt_gmtime() used + # to take the time of day from an unsigned "s % 86400", which wraps for a + # negative time: the header came out as "Thu, 01 Jan 1970 1193046:28:1", + # malformed, with the " GMT" truncated off the end of the buffer. + path = f'{temp_dir}/assets/index.html' + mtime = -14182940 # Sun, 20 Jul 1969 20:17:40 GMT + + try: + os.utime(path, (mtime, mtime)) + except (OSError, OverflowError): + pytest.skip('filesystem rejects a pre-epoch mtime') + + if os.stat(path).st_mtime != mtime: + pytest.skip('filesystem does not keep a pre-epoch mtime') + + last_modified = client.get(url='/index.html')['headers']['Last-Modified'] + + # Checked separately from the equality below: this is the truncation, + # and it is worth failing on its own terms. + assert last_modified.endswith(' GMT'), 'well-formed pre-epoch date' + + # The exact string, not a parsed timestamp: parsedate_to_datetime() + # discards the day name, so a wrong weekday would parse to the right + # instant. This mtime is 164 days pre-epoch, far enough that a weekday + # taken from an unsigned day number comes out wrong. + assert last_modified == formatdate( + mtime, usegmt=True + ), 'pre-epoch mtime is reported as itself, not clamped or wrapped' + + def test_static_redirect(): resp = client.get(url='/dir') assert resp['status'] == 301, 'redirect status' From 5adc3701165016c0b3fbdf03df01bf09dec23b4e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 12 Sep 2026 22:57:22 +0200 Subject: [PATCH 130/391] ci: pin TZ in sanitize.yml's pytest step sanitize.yml ran the Python suite with no TZ, so under UTC test_static_last_modified_is_gmt cannot fail (localtime() and gmtime() agree there) -- the same vacuous-under-UTC problem #328 fixed for build-test.yml. Pin the same Asia/Kolkata zone here so both pytest legs exercise the real assertion instead of the trivially-passing one. No sudo is involved in this step (python3 -m pytest runs directly, and unitd is forked from that process), so a plain step-level env is enough -- unlike build-test.yml's step, which needs sudo -E to carry TZ into unitd. The pin alone is not self-checking: an image without tzdata reads an unknown zone as UTC, where that test skips itself and the leg stays green, so the pin would be silently absent. The "Fail on an empty test run" gate does not catch it either -- it greps [python].*PASSED, which other tests satisfy. So assert the offset before running pytest and fail the leg if the zone did not resolve. Fixes #331. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/sanitize.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 1a2616eea..7569b165a 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -303,11 +303,25 @@ jobs: # graceful tests). - name: Run ${{ matrix.runtime }} sanitizer subset continue-on-error: ${{ matrix.experimental == true }} + # TZ: same pin as build-test.yml's pytest step, and for the same + # reason (see its comment) -- without it this leg's + # test_static_last_modified_is_gmt is as vacuous under UTC as that + # one was. env: ASAN_OPTIONS: "detect_leaks=0" UBSAN_OPTIONS: "print_stacktrace=1" + TZ: Asia/Kolkata run: | set -o pipefail + # An image without tzdata reads an unknown zone as UTC, where the + # test above skips itself and the leg stays green -- the pin would + # be silently absent. Assert the offset instead of trusting it. + offset=$(date +%z) + if [ "$offset" != "+0530" ]; then + echo "TZ=$TZ resolved to $offset, expected +0530:" \ + "the zone is missing from this image and the pin does nothing" + exit 1 + fi python3 -m pytest -v --save-log ${{ matrix.pytest_opts }} \ ${{ matrix.tests }} | tee "${RUNNER_TEMP}/pytest.out" From 054848449a520f59f0dd6a42600524e8475609d6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 12 Sep 2026 23:00:29 +0200 Subject: [PATCH 131/391] test(variables): pin the logged Last-Modified to a known mtime The share case of test_variables_response_header matched the logged $response_header_last_modified with r'.*GMT'. That accepts any instant, so it only proved the variable was substituted at all. It is the reason the local-time rendering fixed in #328 went years unnoticed: the header had the wrong time and a "GMT" label, and this was the only place in the suite that looked at it. Set a fixed mtime with os.utime() and match the exact string email.utils.formatdate(mtime, usegmt=True) produces, so the oracle is computed here rather than read back from the server. The exact string, not a parsed timestamp, for the same reason as test_static.py: a wrong weekday parses to the right instant. $response_header_date stays a shape, since it is the response clock and has no fixed value. No skip guard, unlike test_static_last_modified_is_gmt: that test exists only to tell localtime() and gmtime() apart, so under UTC it proves nothing and says so. This one asserts the correct value in every zone and is worth running everywhere; under a non-UTC zone it additionally catches the localtime skew. Verified it fails against the defect: with src/nxt_http_static.c and src/nxt_gmtime.c checked out from fd2d1501^ and rebuilt, under TZ=Asia/Kolkata, the new form fails ("assert None is not None", no record matching "Sun, 13 Sep 2020 12:26:40 GMT") while the old r'.*GMT' form passes against the same binary. Closes #330. Co-Authored-By: Claude Opus 5 (1M context) --- test/test_variables.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/test_variables.py b/test/test_variables.py index e20a3cd79..379357a1a 100644 --- a/test/test_variables.py +++ b/test/test_variables.py @@ -1,5 +1,7 @@ +import os import re import time +from email.utils import formatdate from pathlib import Path import pytest @@ -544,6 +546,11 @@ def test_variables_response_header(temp_dir, wait_for_record): Path(f'{temp_dir}/foo').mkdir() Path(f'{temp_dir}/foo/index.html').write_text('index', encoding='utf-8') + # A fixed mtime, so the logged $response_header_last_modified can be + # checked against a value computed here instead of against a shape. + mtime = 1600000000 # Sun, 13 Sep 2020 12:26:40 GMT + os.utime(f'{temp_dir}/foo/index.html', (mtime, mtime)) + assert 'success' in client.conf( { "listeners": {"*:8080": {"pass": "routes"}}, @@ -564,10 +571,17 @@ def test_variables_response_header(temp_dir, wait_for_record): '$response_header_connection' ) + # The exact string, not r'.*GMT': matching only the trailing "GMT" + # accepts any instant at all, which is how a Last-Modified rendered from + # localtime() and labelled GMT stayed green here for years. $date is the + # response clock, not a file's, so it stays a shape. + last_modified = re.escape(formatdate(mtime, usegmt=True)) + assert client.get(url='/foo/index.html')['status'] == 200 assert ( wait_for_record( - r'share@.*GMT@".*"@text/html@Unit/.*@.*GMT@5@close', 'access.log' + rf'share@{last_modified}@".*"@text/html@Unit/.*@.*GMT@5@close', + 'access.log', ) is not None ) From 8fbc4836315477f695186d56455eece9f46c50fd Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 12 Sep 2026 23:18:41 +0200 Subject: [PATCH 132/391] types: make nxt_time_t 64-bit on QNX QNX defines time_t as uint32_t and nxt_types.h gave nxt_time_t the signed 32-bit type over it, so no instant past 2038-01-19 is representable: the Date header, a static file's Last-Modified and its ETag all render as a date in 1901 there. The header's own comment has always named the fix. Changes: - nxt_time_t is int64_t under NXT_QNX. - NXT_TIME_T_SIZE is redefined to 8 in the same arm. auto/types probes it from the native sizeof(time_t), but it is read as the size of nxt_time_t: it selects NXT_TIME_T_LEN/HEXLEN/MAX, it gates the Y2038 guard in nxt_time_parse(), and it sizes the gmtime test's range. Redefining it in the header keeps the macro consistent with the type without changing the probe for every other platform. One consequence is deliberate: with 8 the "#if (NXT_TIME_T_SIZE <= 4)" Y2038 guard in nxt_time_parse() compiles out on QNX. That is correct -- the guard rejects values above 0x7FFFFFFF because a 32-bit nxt_time_t cannot hold them, and a 64-bit one can. nxt_time_parse() has no callers in the tree either way. - The static ETag now casts the mtime to nxt_time_t before passing it. nxt_file_mtime() yields a native time_t and "%T" reads an nxt_time_t from the argument list; where the two differ in width that read is wrong. On every platform where they agree the cast is a no-op. - The gmtime test's differential loops stop where the native time_t cannot hold the value. They compare nxt_gmtime() against gmtime(), so they also assume the native type is as wide as nxt_time_t -- an assumption the NXT_TIME_T_SIZE == 8 range breaks on a 4-byte-native platform, which is what QNX becomes here: the loop would hand gmtime() a truncated value and report a failure that is the test's own, not nxt_gmtime()'s. The guard is generic and never triggers where the two types agree. The same file passed &s, an nxt_time_t *, straight to gmtime(), which takes a time_t *: that is a pointer type mismatch wherever the two types differ, so it now converts into a real time_t first. - The #329 comments added in passing are dropped or reworded, and the header no longer states an unsigned 32-bit QNX time_t as timeless fact. QNX SDP 8.0 uses a signed 64-bit time_t, so auto/types already probes 8 there and this whole arm is a no-op; it is load-bearing only for pre-8.0 targets, whose unsigned 32-bit time_t QNX's own 8.0 documentation describes as what previous releases used. Not verified on QNX: there is no QNX host, no QNX CI and no cross compiler, so this is a change to a platform arm that cannot be built here. It was built and tested on Linux both normally and with the QNX arm forced on (make, make tests, ./build/tests: 47 passed, exit 0), which shows the arm compiles and the macro redefinition is clean, not that it is correct on QNX -- on Linux both branches yield a 64-bit nxt_time_t over a 64-bit native time_t, so neither run exercises the width difference at all. Still unknown: whether QNX's own headers or libc interact badly with a 64-bit nxt_time_t anywhere, and whether current QNX still has a 32-bit time_t -- where the probe already yields 8 this change is a no-op. Co-Authored-By: Claude Opus 5 (1M context) --- src/nxt_http_static.c | 16 +++++++--------- src/nxt_types.h | 19 ++++++++++++++++--- src/test/nxt_gmtime_test.c | 38 ++++++++++++++++++++++++++++++-------- 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 9a4c20b14..044464e55 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -701,14 +701,6 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } - /* - * On QNX nxt_time_t is a signed int32_t over an unsigned native - * time_t, so an mtime past 2038 arrives here negative and renders - * as a date in 1901. That is the Y2038 limitation nxt_types.h - * names in its own comment, it costs the Date header and the ETag - * below in the same way, and it cannot be repaired at this call - * site, since nxt_gmtime() takes an nxt_time_t. See #329. - */ nxt_gmtime(nxt_file_mtime(&fi), &tm); field->value = p; @@ -729,8 +721,14 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, } field->value = p; + /* + * nxt_file_mtime() yields a native time_t, which need not be + * nxt_time_t: on QNX it is a 32-bit unsigned type against a 64-bit + * nxt_time_t. "%T" reads an nxt_time_t from the argument list, so + * the value has to be converted before it is passed, not after. + */ field->value_length = nxt_sprintf(p, p + length, "\"%xT-%xO\"", - nxt_file_mtime(&fi), + (nxt_time_t) nxt_file_mtime(&fi), nxt_file_size(&fi)) - p; diff --git a/src/nxt_types.h b/src/nxt_types.h index 723346d9f..e89fd4074 100644 --- a/src/nxt_types.h +++ b/src/nxt_types.h @@ -54,10 +54,23 @@ typedef off_t nxt_off_t; */ #if (NXT_QNX) /* - * QNX defines time_t as uint32_t. - * Y2038 fix: "typedef int64_t nxt_time_t". + * QNX before SDP 8.0 defines time_t as an unsigned 32-bit type, so the + * signed type of the same width this arm used could not represent any + * instant past 2038-01-19. SDP 8.0 and later use a signed 64-bit time_t + * (https://www.qnx.com/developers/docs/8.0/com.qnx.doc.neutrino.prog/topic/ + * timing_Time_functions.html), where this arm matches the generic branch + * and the typedef below is a no-op. + * + * NXT_TIME_T_SIZE is probed by auto/types from the native sizeof(time_t), + * which is 4 on a pre-8.0 QNX, but every user of the macro -- the NXT_TIME_T_* + * macros below, the Y2038 guard in nxt_time_parse(), and the gmtime test -- + * asks it about nxt_time_t rather than about time_t. Redefine it to match + * the type it is read as. */ -typedef int32_t nxt_time_t; +typedef int64_t nxt_time_t; + +#undef NXT_TIME_T_SIZE +#define NXT_TIME_T_SIZE 8 #else /* Y2038, if time_t is 32-bit integer. */ diff --git a/src/test/nxt_gmtime_test.c b/src/test/nxt_gmtime_test.c index 52fbf8e3c..5f9fda0c9 100644 --- a/src/test/nxt_gmtime_test.c +++ b/src/test/nxt_gmtime_test.c @@ -27,13 +27,16 @@ * against gmtime(). Do not "simplify" these into the differential loops * below: handing a nxt_time_t to gmtime(), which takes a time_t, is only * safe where the two types agree, and for a negative value they need not. - * On QNX nxt_time_t is a signed int32_t while the native time_t is uint32_t, - * so the same bits are 1969 to nxt_gmtime() and 2106 to the C library. And - * where nxt_time_t is 8 bytes against a 4-byte time_t -- the case the - * NXT_TIME_T_SIZE test above already contemplates -- the library reads half - * the object: the low half on a little-endian machine, which often looks - * right by accident, and the high half on a big-endian one, which is - * garbage. Fixed fields test the arithmetic on every platform instead. + * Where nxt_time_t is 8 bytes against a 4-byte time_t -- QNX before SDP + * 8.0, whose native time_t is uint32_t, and the case the NXT_TIME_T_SIZE + * test above already contemplates -- the library reads half the object, + * the low half on a little-endian machine, which often looks right by + * accident, and the high half on a big-endian one, which is garbage. + * Fixed fields test the arithmetic on every platform instead. + * + * For the same reason the loops below never hand gmtime() the address of + * the nxt_time_t itself: they convert into a real time_t first, and skip + * the values that do not survive the conversion. * * A file's mtime can be negative ("touch -d 1969-07-20"), and the day-time * step used to wrap and render an hour of 1193046. -432000 and beyond @@ -101,7 +104,19 @@ nxt_gmtime_test(nxt_thread_t *thr) for (s = 0; s < NXT_GMTIME_MAX; s += 86400) { nxt_gmtime(s, &tm0); + native = (time_t) s; + + if ((nxt_time_t) native != s) { + /* + * The native time_t is narrower than nxt_time_t and cannot hold + * s, so gmtime() would be given a truncated value and there is + * nothing left to compare against. Where the two types agree + * this never triggers. + */ + break; + } + tm1 = gmtime(&native); if (tm0.tm_mday != tm1->tm_mday @@ -126,7 +141,13 @@ nxt_gmtime_test(nxt_thread_t *thr) for (s = 0; s < 90000; s++) { nxt_gmtime(s, &tm0); + native = (time_t) s; + + if ((nxt_time_t) native != s) { + break; + } + tm1 = gmtime(&native); if (tm0.tm_hour != tm1->tm_hour @@ -161,7 +182,8 @@ nxt_gmtime_test(nxt_thread_t *thr) start = nxt_thread_monotonic_time(thr); for (s = 0; s < 10000000; s++) { - (void) gmtime(&s); + native = (time_t) s; + (void) gmtime(&native); } nxt_thread_time_update(thr); From ad8f69363307ec744ede3761b41408a010aed66f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 00:33:12 +0200 Subject: [PATCH 133/391] core: remove the fiber implementation nxt_fiber.c has not been compiled since 45c1d41f ("Removed fibers from compilation", 2017-09-22) and nothing was ever added to bring it back. The remaining references were an engine member that was only allocated under "#if 0" (so always NULL), an "if (engine->fibers)" branch that could never be taken, "#if 0" blocks in the thread struct, the log prefix and nxt_sprintf's %PF, and the include that only the dead _setjmp() needed. Deleting it also disposes of the mistyped debug format at nxt_fiber.c:401 ("%T" reading a four-byte nxt_msec_t as an eight-byte nxt_time_t) that reviewers flagged on PR #334; the file was never compile-checked, so the mismatch was latent rather than reachable. This matches the upstream maintainer's own disposition: hongzhidao named nxt_fiber.c as removable "at the next big version" in nginx/unit#740 (2022) and removed it in his unit-1.26 branch (commits 4d2ea11f and f662c394, 2026-08-01). Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 6 + docs/changes.xml | 10 + src/nxt_app_log.c | 20 -- src/nxt_event_engine.c | 25 --- src/nxt_event_engine.h | 5 - src/nxt_fiber.c | 465 ----------------------------------------- src/nxt_fiber.h | 57 ----- src/nxt_main.h | 1 - src/nxt_router.c | 3 - src/nxt_runtime.c | 3 - src/nxt_sprintf.c | 7 - src/nxt_thread.h | 8 - src/nxt_unix.h | 1 - 13 files changed, 16 insertions(+), 595 deletions(-) delete mode 100644 src/nxt_fiber.c delete mode 100644 src/nxt_fiber.h diff --git a/CHANGES b/CHANGES index fb3dbfce3..af7f2489f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Change: the fiber implementation has been removed. It had not been + compiled since 2017: the only allocation of the engine's fiber + stack sat under "#if 0", so the pointer was always NULL and the + setjmp() branch that tested it could not run. The thread, log and + sprintf sites were already disabled. Nothing observable changes. + *) Bugfix: do not exit when the capget() syscall is denied. A syscall filter that omits capget -- a systemd "SystemCallFilter=" allowlist, a hand-written seccomp profile -- made a non-root unitd fail to start diff --git a/docs/changes.xml b/docs/changes.xml index 635a48b89..55359ea88 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,16 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +the fiber implementation has been removed. It had not been compiled since +2017: the only allocation of the engine's fiber stack sat under "#if 0", so +the pointer was always NULL and the setjmp() branch that tested it could not +run. The thread, log and sprintf sites were already disabled. Nothing +observable changes. + + + do not exit when the capget() syscall is denied. A syscall filter that diff --git a/src/nxt_app_log.c b/src/nxt_app_log.c index ae57c2a2f..7d7d4f8e8 100644 --- a/src/nxt_app_log.c +++ b/src/nxt_app_log.c @@ -41,28 +41,8 @@ nxt_log_time_handler(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...) syslogmsg = p; #endif -#if 0 - nxt_fid_t fid; - const char *id; - nxt_fiber_t *fib; - - fib = nxt_fiber_self(thr); - - if (fib != NULL) { - id = "[%V] %PI#%PT#%PF "; - fid = nxt_fiber_id(fib); - - } else { - id = "[%V] %PI#%PT "; - fid = 0; - } - - p = nxt_sprintf(p, end, id, &nxt_log_levels[level], nxt_pid, - nxt_thread_tid(thr), fid); -#else p = nxt_sprintf(p, end, "[%V] %PI#%PT ", &nxt_log_levels[level], nxt_pid, nxt_thread_tid(thr)); -#endif if (log->ident != 0) { p = nxt_sprintf(p, end, "*%D ", log->ident); diff --git a/src/nxt_event_engine.c b/src/nxt_event_engine.c index 079588646..3d5dea8a6 100644 --- a/src/nxt_event_engine.c +++ b/src/nxt_event_engine.c @@ -62,15 +62,6 @@ nxt_event_engine_create(nxt_task_t *task, engine->batch = batch; -#if 0 - if (flags & NXT_ENGINE_FIBERS) { - engine->fibers = nxt_fiber_main_create(engine); - if (engine->fibers == NULL) { - goto fibers_fail; - } - } -#endif - engine->current_work_queue = &engine->fast_work_queue; nxt_work_queue_cache_create(&engine->work_queue_cache, 0); @@ -153,12 +144,6 @@ nxt_event_engine_create(nxt_task_t *task, nxt_free(engine->signals); nxt_work_queue_cache_destroy(&engine->work_queue_cache); - nxt_free(engine->fibers); - -#if 0 -fibers_fail: -#endif - nxt_free(engine); return NULL; @@ -529,16 +514,6 @@ nxt_event_engine_start(nxt_event_engine_t *engine) thr = nxt_thread(); - if (engine->fibers) { - /* - * _setjmp() cannot be wrapped in a function since return from - * the function clobbers stack used by future _setjmp() returns. - */ - _setjmp(engine->fibers->fiber.jmp); - - /* A return point from fibers. */ - } - thr->log = engine->task.log; for ( ;; ) { diff --git a/src/nxt_event_engine.h b/src/nxt_event_engine.h index cff72022f..77ed1de3c 100644 --- a/src/nxt_event_engine.h +++ b/src/nxt_event_engine.h @@ -407,9 +407,6 @@ void nxt_fd_event_hash_destroy(nxt_lvlhsh_t *lvlhsh); (engine)->event.enable_accept(engine, ev) -#define NXT_ENGINE_FIBERS 1 - - typedef struct { nxt_fd_t fds[2]; nxt_fd_event_t event; @@ -465,8 +462,6 @@ struct nxt_event_engine_s { nxt_event_signals_t *signals; - nxt_fiber_main_t *fibers; - /* The engine ID, the main engine has ID 0. */ uint32_t id; diff --git a/src/nxt_fiber.c b/src/nxt_fiber.c deleted file mode 100644 index d6cac8930..000000000 --- a/src/nxt_fiber.c +++ /dev/null @@ -1,465 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static char *nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib); -static void nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent); -static void nxt_fiber_switch_handler(nxt_task_t *task, void *obj, void *data); -static void nxt_fiber_switch(nxt_task_t *task, nxt_fiber_t *fib); -static void nxt_fiber_timer_handler(nxt_task_t *task, void *obj, void *data); - - -#define nxt_fiber_enqueue(thr, task, fib) \ - nxt_work_queue_add(&(thr)->engine->fast_work_queue, \ - nxt_fiber_switch_handler, task, fib, NULL) - - -nxt_fiber_main_t * -nxt_fiber_main_create(nxt_event_engine_t *engine) -{ - nxt_fiber_main_t *fm; - - fm = nxt_zalloc(sizeof(nxt_fiber_main_t)); - if (nxt_slow_path(fm == NULL)) { - return NULL; - } - - fm->engine = engine; - fm->stack_size = 512 * 1024 - nxt_pagesize; - fm->idle = NULL; - - return fm; -} - - -nxt_int_t -nxt_fiber_create(nxt_fiber_start_t start, void *data, size_t stack) -{ - int ret; - jmp_buf parent; - nxt_fid_t fid; - nxt_fiber_t *fib; - nxt_thread_t *thr; - nxt_fiber_main_t *fm; - - thr = nxt_thread(); - fm = thr->engine->fibers; - - fid = ++fm->fid; - - if (fid == 0) { - fid = ++fm->fid; - } - - fib = fm->idle; - - if (fib != NULL) { - fm->idle = fib->next; - fib->fid = fid; - fib->start = start; - fib->data = data; - fib->main = fm; - - fib->task.thread = thr; - fib->task.log = thr->log; - fib->task.ident = nxt_task_next_ident(); - - nxt_debug(&fib->task, "fiber create cached: %PF", fib->fid); - - nxt_fiber_enqueue(thr, &fm->engine->task, fib); - - return NXT_OK; - } - - nxt_log_debug(thr->log, "fiber create"); - - fib = nxt_malloc(sizeof(nxt_fiber_t)); - if (nxt_slow_path(fib == NULL)) { - return NXT_ERROR; - } - - fib->fid = fid; - fib->start = start; - fib->data = data; - fib->stack_size = fm->stack_size; - fib->main = fm; - - fib->task.thread = thr; - fib->task.log = thr->log; - fib->task.ident = nxt_task_next_ident(); - - fib->stack = nxt_fiber_create_stack(&fib->task, fib); - - if (nxt_fast_path(fib->stack != NULL)) { - - if (_setjmp(parent) != 0) { - nxt_log_debug(thr->log, "fiber create: %PF", fib->fid); - return NXT_OK; - } - - nxt_fiber_switch_stack(fib, &parent); - /* It does not return if the switch was successful. */ - } - - ret = munmap(fib->stack - nxt_pagesize, fib->stack_size + nxt_pagesize); - - if (nxt_slow_path(ret != 0)) { - nxt_log_alert(thr->log, "munmap() failed %E", nxt_errno); - } - - nxt_free(fib); - - return NXT_ERROR; -} - - -#if (NXT_LINUX) - -static char * -nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib) -{ - char *s; - size_t size; - - size = fib->stack_size + nxt_pagesize; - - s = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | MAP_GROWSDOWN, -1, 0); - - if (nxt_slow_path(s == MAP_FAILED)) { - nxt_alert(task, "fiber stack " - "mmap(%uz, MAP_PRIVATE|MAP_ANON|MAP_GROWSDOWN) failed %E", - size, nxt_errno); - - return NULL; - } - - if (nxt_slow_path(mprotect(s, nxt_pagesize, PROT_NONE) != 0)) { - nxt_alert(task, "fiber stack mprotect(%uz, PROT_NONE) failed %E", - size, nxt_errno); - - return NULL; - } - - s += nxt_pagesize; - - nxt_debug(task, "fiber stack mmap: %p", s); - - return s; -} - -#else /* Generic version. */ - -static char * -nxt_fiber_create_stack(nxt_task_t *task, nxt_fiber_t *fib) -{ - char *s; - size_t size; - - size = fib->stack_size + nxt_pagesize; - - s = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - - if (nxt_slow_path(s == MAP_FAILED)) { - nxt_alert(task, "fiber stack mmap(%uz, MAP_PRIVATE|MAP_ANON) failed %E", - size, nxt_errno); - - return NULL; - } - - if (nxt_slow_path(mprotect(s, nxt_pagesize, PROT_NONE) != 0)) { - nxt_alert(task, "fiber stack mprotect(%uz, PROT_NONE) failed %E", - size, nxt_errno); - - return NULL; - } - - s += nxt_pagesize; - - nxt_debug(task, "fiber stack mmap: %p", s); - - return s; -} - -#endif - - -#if (NXT_LINUX && NXT_64BIT) - -/* - * Linux 64-bit ucontext version. 64-bit glibc makecontext() passes - * pointers as signed int's. The bug has been fixed in glibc 2.8. - */ - -static void nxt_fiber_trampoline(uint32_t fh, uint32_t fl, uint32_t ph, - uint32_t pl); - - -static void -nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent) -{ - ucontext_t uc; - - nxt_debug(&fib->task, "fiber switch to stack: %p", fib->stack); - - if (nxt_slow_path(getcontext(&uc) != 0)) { - nxt_alert(&fib->task, "getcontext() failed"); - return; - } - - uc.uc_link = NULL; - uc.uc_stack.ss_sp = fib->stack; - uc.uc_stack.ss_size = fib->stack_size; - - makecontext(&uc, (void (*)(void)) nxt_fiber_trampoline, 4, - (uint32_t) ((uintptr_t) fib >> 32), - (uint32_t) ((uintptr_t) fib & 0xFFFFFFFF), - (uint32_t) ((uintptr_t) parent >> 32), - (uint32_t) ((uintptr_t) parent & 0xFFFFFFFF)); - - setcontext(&uc); - - nxt_alert(&fib->task, "setcontext() failed"); -} - - -static void -nxt_fiber_trampoline(uint32_t fh, uint32_t fl, uint32_t ph, uint32_t pl) -{ - jmp_buf *parent; - nxt_task_t *task; - nxt_fiber_t *fib; - - fib = (nxt_fiber_t *) (((uintptr_t) fh << 32) + fl); - parent = (jmp_buf *) (((uintptr_t) ph << 32) + pl); - - task = &fib->task; - - if (_setjmp(fib->jmp) == 0) { - nxt_debug(task, "fiber return to parent stack"); - - nxt_fiber_enqueue(task->thread, task, fib); - - _longjmp(*parent, 1); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber start"); - - fib->start(fib->data); - - nxt_fiber_exit(task, &fib->main->fiber, NULL); - - nxt_unreachable(); -} - -#elif (NXT_HAVE_UCONTEXT) - -/* Generic ucontext version. */ - -static void nxt_fiber_trampoline(nxt_fiber_t *fib, jmp_buf *parent); - - -static void -nxt_fiber_switch_stack(nxt_fiber_t *fib, jmp_buf *parent) -{ - ucontext_t uc; - - nxt_debug(&fib->task, "fiber switch to stack: %p", fib->stack); - - if (nxt_slow_path(getcontext(&uc) != 0)) { - nxt_alert(&fib->task, "getcontext() failed"); - return; - } - - uc.uc_link = NULL; - uc.uc_stack.ss_sp = fib->stack; - uc.uc_stack.ss_size = fib->stack_size; - - makecontext(&uc, (void (*)(void)) nxt_fiber_trampoline, 2, fib, parent); - - setcontext(&uc); - -#if !(NXT_SOLARIS) - /* Solaris declares setcontext() as __NORETURN. */ - - nxt_alert(&fib->task, "setcontext() failed"); -#endif -} - - -static void -nxt_fiber_trampoline(nxt_fiber_t *fib, jmp_buf *parent) -{ - nxt_task_t *task; - - task = &fib->task; - - if (_setjmp(fib->jmp) == 0) { - nxt_debug(task, "fiber return to parent stack"); - - nxt_fiber_enqueue(task->thread, task, fib); - - _longjmp(*parent, 1); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber start"); - - fib->start(fib->data); - - nxt_fiber_exit(task, &fib->main->fiber, NULL); - - nxt_unreachable(); -} - -#else - -#error No ucontext(3) interface. - -#endif - - -static void -nxt_fiber_switch_handler(nxt_task_t *task, void *obj, void *data) -{ - nxt_fiber_t *fib; - - fib = obj; - - nxt_fiber_switch(task, fib); - nxt_unreachable(); -} - - -static void -nxt_fiber_switch(nxt_task_t *task, nxt_fiber_t *fib) -{ - nxt_debug(task, "fiber switch: %PF", fib->fid); - - task->thread->fiber = fib; - - _longjmp(fib->jmp, 1); - - nxt_unreachable(); -} - - -nxt_fiber_t * -nxt_fiber_self(nxt_thread_t *thr) -{ - return (nxt_fast_path(thr != NULL)) ? thr->fiber : NULL; -} - - -void -nxt_fiber_yield(nxt_task_t *task) -{ - nxt_fiber_t *fib; - - fib = task->thread->fiber; - - if (_setjmp(fib->jmp) == 0) { - - nxt_debug(task, "fiber yield"); - - nxt_fiber_enqueue(task->thread, &fib->main->engine->task, fib); - - nxt_fiber_switch(task, &fib->main->fiber); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber yield return"); -} - - -void -nxt_fiber_sleep(nxt_task_t *task, nxt_msec_t timeout) -{ - nxt_fiber_t *fib; - - fib = task->thread->fiber; - - fib->timer.work_queue = &task->thread->engine->fast_work_queue; - fib->timer.handler = nxt_fiber_timer_handler; - fib->timer.log = &nxt_main_log; - - task = &fib->task; - - nxt_timer_add(task->thread->engine, &fib->timer, timeout); - - if (_setjmp(fib->jmp) == 0) { - - nxt_debug(task, "fiber sleep: %T", timeout); - - nxt_fiber_switch(task, &fib->main->fiber); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber sleep return"); -} - - -static void -nxt_fiber_timer_handler(nxt_task_t *task, void *obj, void *data) -{ - nxt_fiber_t *fib; - nxt_timer_t *ev; - - ev = obj; - - nxt_debug(task, "fiber timer handler"); - - fib = nxt_timer_data(ev, nxt_fiber_t, timer); - - nxt_fiber_switch(task, fib); - - nxt_unreachable(); -} - - -void -nxt_fiber_wait(nxt_task_t *task) -{ - nxt_fiber_t *fib; - - fib = task->thread->fiber; - - if (_setjmp(fib->jmp) == 0) { - nxt_debug(task, "fiber wait"); - - nxt_fiber_switch(task, &fib->main->fiber); - - nxt_unreachable(); - } - - nxt_debug(task, "fiber wait return"); -} - - -void -nxt_fiber_exit(nxt_task_t *task, nxt_fiber_t *next, void *data) -{ - nxt_fiber_t *fib; - - fib = task->thread->fiber; - - nxt_debug(task, "fiber exit"); - - /* TODO: limit idle fibers. */ - fib->next = fib->main->idle; - fib->main->idle = fib; - - nxt_fiber_switch(task, next); - - nxt_unreachable(); -} diff --git a/src/nxt_fiber.h b/src/nxt_fiber.h deleted file mode 100644 index 7f1beace6..000000000 --- a/src/nxt_fiber.h +++ /dev/null @@ -1,57 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_FIBER_H_INCLUDED_ -#define _NXT_FIBER_H_INCLUDED_ - - -typedef struct nxt_fiber_main_s nxt_fiber_main_t; -typedef void (*nxt_fiber_start_t)(void *data); - - -typedef uint32_t nxt_fid_t; -#define nxt_fiber_id(f) (f)->fid; - - -typedef struct nxt_fiber_s nxt_fiber_t; - -struct nxt_fiber_s { - jmp_buf jmp; - nxt_fid_t fid; - nxt_fiber_start_t start; - void *data; - char *stack; - size_t stack_size; - nxt_err_t err; - - nxt_task_t task; - - nxt_fiber_main_t *main; - nxt_fiber_t *next; - - nxt_timer_t timer; -}; - - -struct nxt_fiber_main_s { - nxt_fiber_t fiber; - nxt_fiber_t *idle; - nxt_event_engine_t *engine; - size_t stack_size; - nxt_fid_t fid; -}; - - -nxt_fiber_main_t *nxt_fiber_main_create(nxt_event_engine_t *engine); -nxt_int_t nxt_fiber_create(nxt_fiber_start_t start, void *data, size_t stack); -void nxt_fiber_yield(nxt_task_t *task); -void nxt_fiber_sleep(nxt_task_t *task, nxt_msec_t timeout); -void nxt_fiber_wait(nxt_task_t *task); -void nxt_fiber_exit(nxt_task_t *task, nxt_fiber_t *next, void *data); -NXT_EXPORT nxt_fiber_t *nxt_fiber_self(nxt_thread_t *thr); - - -#endif /* _NXT_FIBER_H_INCLUDED_ */ diff --git a/src/nxt_main.h b/src/nxt_main.h index 7880e55f8..c0731ea7a 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -55,7 +55,6 @@ typedef uint16_t nxt_port_id_t; #include #include #include -#include #include #include #include diff --git a/src/nxt_router.c b/src/nxt_router.c index 0517ac432..210d3c8e1 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -4607,9 +4607,6 @@ nxt_router_thread_start(void *data) engine->task.log = thread->log; thread->engine = engine; thread->task = &engine->task; -#if 0 - thread->fiber = &engine->fibers->fiber; -#endif engine->mem_pool = nxt_mp_create(4096, 128, 1024, 64); if (nxt_slow_path(engine->mem_pool == NULL)) { diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index e8dcabfa6..0f87082b7 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -301,9 +301,6 @@ nxt_runtime_event_engines(nxt_task_t *task, nxt_runtime_t *rt) thread = task->thread; thread->engine = engine; -#if 0 - thread->fiber = &engine->fibers->fiber; -#endif engine->id = rt->last_engine_id++; engine->mem_pool = nxt_mp_create(1024, 128, 256, 32); diff --git a/src/nxt_sprintf.c b/src/nxt_sprintf.c index 2e29e80ef..8b1db8b25 100644 --- a/src/nxt_sprintf.c +++ b/src/nxt_sprintf.c @@ -28,7 +28,6 @@ * * %PI nxt_pid_t, process id * %PT nxt_tid_t, thread id - * %PF nxt_fid_t, fiber id * %PH pthread_t handle returned by pthread_self() * * %s null-terminated string @@ -507,12 +506,6 @@ nxt_vsprintf(u_char *buf, u_char *end, const char *fmt, va_list args) ui64 = (uint64_t) (uintptr_t) va_arg(args, nxt_tid_t); sign = 0; goto number; -#if 0 - case 'F': - ui64 = (uint64_t) va_arg(args, nxt_fid_t); - sign = 0; - goto number; -#endif case 'H': ui64 = (uint64_t) (uintptr_t) va_arg(args, pthread_t); spf.hex = HEXADECIMAL; diff --git a/src/nxt_thread.h b/src/nxt_thread.h index 53b2d8c0f..80fdd477d 100644 --- a/src/nxt_thread.h +++ b/src/nxt_thread.h @@ -148,14 +148,6 @@ struct nxt_thread_s { nxt_event_engine_t *engine; void *data; -#if 0 - /* - * Although pointer to a current fiber should be a property of - * engine->fibers, its placement here eliminates 2 memory accesses. - */ - nxt_fiber_t *fiber; -#endif - nxt_random_t random; }; diff --git a/src/nxt_unix.h b/src/nxt_unix.h index 6bc6be5ee..cf2402d36 100644 --- a/src/nxt_unix.h +++ b/src/nxt_unix.h @@ -153,7 +153,6 @@ #include #include #include -#include #include #include #include From 9e20af27e527ea602ae246a757a4f1d9da221351 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 01:33:00 +0200 Subject: [PATCH 134/391] core: remove the source files the build never uses src/nxt_job_cache_file.c is not named anywhere in auto/, so no configuration ever compiled it. Nothing in the tree referenced the file or its symbols, and the type it allocates is not declared anywhere, so it would not have compiled had the build picked it up. src/nxt_source.h is removed because nothing includes it. Of the 106 headers in src/, it is the only one with no #include of it anywhere in the tree. Nothing observable changes: no compiled line is affected. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 7 +++++++ docs/changes.xml | 11 +++++++++++ src/nxt_job_cache_file.c | 24 ----------------------- src/nxt_source.h | 42 ---------------------------------------- 4 files changed, 18 insertions(+), 66 deletions(-) delete mode 100644 src/nxt_job_cache_file.c delete mode 100644 src/nxt_source.h diff --git a/CHANGES b/CHANGES index af7f2489f..a4bb5989f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Change: two source files that the build never used have been + removed. src/nxt_job_cache_file.c is named nowhere in auto/, so no + configuration ever compiled it, and nothing in the tree referenced + it or its symbols. src/nxt_source.h was removed because nothing + includes it: of the 106 headers in src/, it was the only one with + no #include anywhere in the tree. Nothing observable changes. + *) Change: the fiber implementation has been removed. It had not been compiled since 2017: the only allocation of the engine's fiber stack sat under "#if 0", so the pointer was always NULL and the diff --git a/docs/changes.xml b/docs/changes.xml index 55359ea88..28eaac7b1 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,17 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +two source files that the build never used have been removed. +src/nxt_job_cache_file.c is named nowhere in auto/, so no configuration ever +compiled it, and nothing in the tree referenced it or its symbols. +src/nxt_source.h was removed because nothing includes it: of the 106 headers +in src/, it was the only one with no #include anywhere in the tree. Nothing +observable changes. + + + the fiber implementation has been removed. It had not been compiled since diff --git a/src/nxt_job_cache_file.c b/src/nxt_job_cache_file.c deleted file mode 100644 index 13f01a453..000000000 --- a/src/nxt_job_cache_file.c +++ /dev/null @@ -1,24 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - - -#include - - -nxt_job_cache_file_t * -nxt_job_cache_file_create(nxt_mp_t *mp) -{ - nxt_job_cache_file_t *jbc; - - jbc = nxt_job_create(mp, sizeof(nxt_job_cache_file_t)); - - if (nxt_fast_path(jbc != NULL)) { - jbc->file.fd = NXT_FILE_INVALID; - jbc->read_required = nxt_job_file_read_required; - } - - return jbc; -} diff --git a/src/nxt_source.h b/src/nxt_source.h deleted file mode 100644 index 0b4658dd7..000000000 --- a/src/nxt_source.h +++ /dev/null @@ -1,42 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_SOURCE_H_INCLUDED_ -#define _NXT_SOURCE_H_INCLUDED_ - - -/* - * A source handler should store a pointer to a passed source hook, but not - * the hook's values themselves, because a source filter may change the values. - */ -typedef struct { - void *context; - nxt_work_handler_t filter; -} nxt_source_hook_t; - - -typedef void (*nxt_source_handler_t)(void *source_context, - nxt_source_hook_t *query); - - -#define nxt_source_filter(thr, wq, task, next, out) \ - do { \ - if (thr->engine->batch != 0) { \ - nxt_thread_work_queue_add(thr, wq, nxt_source_filter_handler, \ - task, next, out); \ - \ - } else { \ - (next)->filter(task, (next)->context, out); \ - } \ - \ - } while (0) - - -NXT_EXPORT void nxt_source_filter_handler(nxt_task_t *task, void *obj, - void *data); - - -#endif /* _NXT_SOURCE_H_INCLUDED_ */ From 45c6c4e653f3303c066a21aa70bad65198d4c75d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 06:16:09 +0200 Subject: [PATCH 135/391] application: report a worker that dies before it is created An application worker is a child of the prototype, so nxt_proto_sigchld_handler() reaps it, and that reaper does not notify anyone about a child still in the CREATING state. Nothing else covers it: the router registers an on-demand start with no RPC peer, main is not the parent and gets no SIGCHLD, and before PROCESS_CREATED there is no port to the worker. The start RPC therefore stays armed forever and app->pending_processes keeps its slot. Once the application reaches its "max", nxt_router_app_can_start() is false for good -- the application never starts another process, and a reload only recovers if the application's own configuration text changed. Because an application's "timeout" defaults to 0, the requests waiting on it do not even fail: they hang. Reproduced by killing a worker on either side of its PROCESS_CREATED write, with nothing else changed. Dying after it: 503 in 0.59 s, processes.starting back to 0, the next request served. Dying before it: the request hangs until the client gives up, processes.starting stuck at 1, and every later request hangs too. The gate stays. It is there because the pid is the whole content of REMOVE_PID and, until the PROCESS_CREATED exchange completes, the prototype does not necessarily know a globally valid one -- under pid isolation process->pid is still the namespace-local pid that fork() returned (900828cc). Broadcasting it would ask every receiver to remove whatever unrelated process happens to hold that number. So answer the initiator directly instead. nxt_proto_child_exited() keeps the existing REMOVE_PID path verbatim for a child that was announced, and for one that was not it writes an RPC error to the port the start request came from, addressed by process->stream_pid and ->stream_port -- recorded from the START_PROCESS message next to ->stream -- rather than by any pid of the dead child. It is the same message the fork-failure path already sends. The test drives both states against one router port and requires a message to reach the initiator for each. It is red without the fix ("nothing was written to the start initiator"), in release and debug builds alike. The case where the *prototype* dies part-way through an on-demand worker start is the same class and is still open; it needs the router to arm that RPC with a peer, as the prefork path does. Answering the start initiator is not the whole job. A worker that got as far as WHOAMI made main create a process record and a port for it, holding main's end of the worker's port socket (nxt_main_process_whoami_handler()), linked into the prototype's ->children; the router can hold one too. REMOVE_PID is what retires them while the prototype runs, and the CREATING branch does not send it. Until this change that leak was capped by the wedge itself: the application stopped starting processes, so at most one record could be stranded. Now that the start RPC is retired and requests retry, a worker that keeps dying in that window costs main one record and one descriptor per attempt, for as long as that prototype lives. It is not permanent: main reclaims the whole set when it reaps the prototype and walks its ->children. Measured with a fault latch inside the window, main's descriptor count went 12 -> 21 over 8 failing starts and 41 over 28, reclaimed only when the prototype was killed. With the cleanup it goes 12 -> 13 and stays there across 58. Send the notification after ->stream has been cleared, so it carries no stream -- the initiator has already been answered directly, and a stream-bearing REMOVE_PID would make nxt_router_remove_pid_handler() fail the same RPC a second time. Gate it on rt->is_pid_isolated, the same test nxt_process_create() uses to decide whether a forked child's pid may enter the runtime hash at all. When it is clear, the prototype shares main's pid namespace and ->pid is by construction the number main read from SCM_CREDENTIALS on the WHOAMI message, so the pid is a usable global key. When it is set there is no safe pid to send: a prototype's namespace-local counter climbs with every worker it forks, straight into the range the daemon's own pids occupy, so broadcasting it would ask every receiver to remove an unrelated process. Dropping a live sibling's ports loses requests, which is worse than the leak, so that case is logged and left; the comment records how main can close it later from the other side, via the NSpid line of /proc//status, without trusting a sender-supplied pid. The test now covers both: the isolated case must produce exactly one message to the initiator, and removing the guard makes it red. The RPC_ERROR write result is not discarded. The failure that reaches this call site is an allocation failure in nxt_port_msg_chk_insert(), and it leaves no message behind: the prototype maps no queue for a router port -- nxt_app_new_port_handler() closes the queue descriptor every NEW_PORT carries, and main maps a queue only for an application port -- so nxt_port_socket_write2() never takes the shared-ring path that answers NXT_AGAIN. Clearing the stream regardless made the following REMOVE_PID streamless, nxt_router_remove_pid_handler() returned early, and the start RPC stayed armed: the wedge this function exists to close, re-entered on its own failure path and with no log line. The stream is cleared only on NXT_OK; otherwise it rides the REMOVE_PID and the router converts that into the same error, the fallback the CREATED path has always used. That fallback allocates from the same pools and can fail the same way, and then only "limits.start_timeout" retires the start, so the case is alerted rather than left silent. Under pid isolation nothing can carry it, so that case is logged and cleared. The alerts are NXT_LOG_WARN while the prototype is itself exiting, where a CREATING child whose initiator is already gone is the expected shape of a teardown rather than an anomaly of one; and the "died before it was created" line no longer says "(isolated %PI)" for a process that is not isolated, where the global pid is the number an operator needs. Two comments are corrected rather than carried over. Both said the router registers the on-demand start RPC with no peer, so REMOVE_PID for a dying prototype recovered nothing. That stopped being true: nxt_router_start_app_process_handler() now calls nxt_port_rpc_ex_set_peer() (src/nxt_router.c), so the prototype's own death is already answered, and nxt_application.c said as much a few hundred lines above the block that denied it. What is left is this function's case alone. The NSpid fix that would close it properly is tracked in #310. Co-Authored-By: Claude Opus 5 --- CHANGES | 6 + auto/sources | 1 + docs/changes.xml | 10 + src/nxt_application.c | 243 +++++++++- src/nxt_application.h | 3 + src/nxt_process.h | 10 + src/test/nxt_proto_creating_wedge_test.c | 594 +++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 9 files changed, 869 insertions(+), 3 deletions(-) create mode 100644 src/test/nxt_proto_creating_wedge_test.c diff --git a/CHANGES b/CHANGES index a4bb5989f..9e7845d69 100644 --- a/CHANGES +++ b/CHANGES @@ -23,6 +23,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 the "credential" namespace, and applications run as unitd's own uid. Every other capget() failure remains fatal. + *) Bugfix: an application worker that died before it finished starting + was reported to nobody, so the start it was forked for stayed pending + forever. Once the application reached its "processes" maximum it + started no further process, and because an application's "timeout" + defaults to 0 the requests waiting on it hung instead of failing. + *) Security: application processes no longer inherit unitd's capabilities. A unitd started as a non-root user that had been granted capabilities -- systemd "AmbientCapabilities=", file diff --git a/auto/sources b/auto/sources index 738a5b68c..f851ed4d3 100644 --- a/auto/sources +++ b/auto/sources @@ -189,6 +189,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ + src/test/nxt_proto_creating_wedge_test.c \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_ctrunc_test.c \ src/test/nxt_port_fd_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 28eaac7b1..344743ab6 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -44,6 +44,16 @@ remains fatal. + + +an application worker that died before it finished starting was reported to +nobody, so the start it was forked for stayed pending forever. Once the +application reached its "processes" maximum it started no further process, +and because an application's "timeout" defaults to 0 the requests waiting on +it hung instead of failing. + + + application processes no longer inherit unitd's capabilities. A unitd diff --git a/src/nxt_application.c b/src/nxt_application.c index 3115492ac..c94cec238 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -807,7 +807,16 @@ nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) process->user_cred = &rt->user_cred; process->data.app = nxt_app_conf; + + /* + * Remember who to answer, not just which stream: the RPC is registered on + * the initiator's port, and by the time this child is reaped the message + * that named it is long gone. nxt_proto_child_exited() is then the only + * thing left that can report a child which died before it was announced. + */ process->stream = msg->port_msg.stream; + process->stream_pid = msg->port_msg.pid; + process->stream_port = msg->port_msg.reply_port; init->siblings = &nxt_proto_children; @@ -1174,9 +1183,7 @@ nxt_proto_sigchld_handler(nxt_task_t *task, void *obj, void *data) port = nxt_process_port_first(process); } - if (process->state != NXT_PROCESS_STATE_CREATING) { - nxt_port_remove_notify_others(task, process); - } + nxt_proto_child_exited(task, process); nxt_process_close_ports(task, process); @@ -1192,6 +1199,236 @@ nxt_proto_sigchld_handler(nxt_task_t *task, void *obj, void *data) } +/* + * Report a reaped child of the prototype. Every child is forked to satisfy a + * START_PROCESS request, and that request has an RPC handler armed on the + * initiator's port; something has to retire it when the child dies or the + * initiator waits forever. This closes the case where the child dies. + * + * The case where the *prototype* dies part-way through an on-demand start is + * closed elsewhere and no longer belongs on this list: + * nxt_router_start_app_process_handler() keys that RPC on the prototype + * (nxt_port_rpc_ex_set_peer(), src/nxt_router.c), so the REMOVE_PID for the + * prototype reaches nxt_port_rpc_remove_peer() and fails the start, releasing + * the app->pending_processes slot with it. + * + * What that leaves is this function's case alone: a child that dies before + * PROCESS_CREATED, which no REMOVE_PID describes, because until the handshake + * completes the prototype does not necessarily know a globally valid pid for + * it. + * + * A child that got as far as PROCESS_CREATED is reported by REMOVE_PID, which + * carries ->stream: nxt_router_remove_pid_handler() turns a stream-bearing + * REMOVE_PID into an RPC error. A child still in the CREATING state cannot be + * announced that way. The pid is the whole content of REMOVE_PID, and until + * the PROCESS_CREATED exchange completes the prototype does not necessarily + * know a globally valid one: under pid isolation ->pid is still the + * namespace-local pid nxt_process_create() got from fork(), and the global pid + * only arrives with PROCESS_CREATED (see nxt_proto_process_created_handler() + * and 900828cc, which is why such a process is deliberately kept out of the + * global pid hash). Broadcasting that pid would ask every receiver to remove + * whatever unrelated process happens to hold it. + * + * So the gate stays exactly as it was, and the CREATING case is answered + * directly instead: an RPC error to the port the start request came from, + * addressed by ->stream_pid/->stream_port rather than by any pid of the dead + * child. It is the same message nxt_proto_start_process_handler() sends when + * the fork itself fails. + * + * Answering the initiator is not the whole job, though, because the initiator + * is not the only process left holding the child. A worker that got as far as + * WHOAMI made main create a process record and a port for it, with main's end + * of the worker's port socket in it (nxt_main_process_whoami_handler()), and + * linked that record into the prototype's ->children. REMOVE_PID is what + * retires it -- nxt_proc_remove_notify_matrix pairs a dying APP with MAIN -- + * so a CREATING child that is only answered on the RPC leaves main holding a + * record and an fd until the prototype itself exits. Before this whole fix + * that leak was capped by the wedge: the application stopped starting + * processes, so at most one could leak. Now that the start RPC is retired and + * requests retry, a worker that keeps dying in that window costs main one + * record and one descriptor per attempt, without bound. + * + * The pid problem is the same one the gate exists for, so the answer is the + * same test the rest of the code already uses for "is this pid a usable global + * key": rt->is_pid_isolated, which nxt_process_create() consults to decide + * whether a forked child may go into the runtime hash at all. When it is + * clear, the prototype shares main's pid namespace, ->pid is the fork() return + * in that namespace, and it is by construction the same number main read from + * SCM_CREDENTIALS on the WHOAMI message -- so REMOVE_PID is safe. The + * notification is deliberately made after ->stream has been cleared, so it + * carries no stream: the initiator has already been answered directly, and a + * stream-bearing REMOVE_PID would make nxt_router_remove_pid_handler() fail + * the same RPC a second time. + * + * Ordering is not a race even though the two messages come from two processes: + * the worker's WHOAMI and the prototype's REMOVE_PID are both written to the + * single write end of main's port socketpair, inherited by every descendant, + * so they share one kernel queue. The worker's WHOAMI write completes before + * it exits, and the prototype writes only after waitpid() has reaped it. If + * the WHOAMI never reached the socket it died with the worker, and main has no + * record to retire. + * + * When rt->is_pid_isolated is set there is no safe pid to send: ->pid is the + * namespace-local one, and the global pid main and the router keyed their + * records on cannot be derived from it here. Sending it anyway would ask + * every receiver to remove whatever unrelated process holds that number -- and + * a prototype's namespace-local counter climbs with each worker it forks, so + * it walks into the range the daemon's own pids occupy. Removing a live + * sibling's ports drops requests, which is worse than the leak, so that case + * is left alone and logged. + * + * Closing it belongs on the other side. The prototype cannot map its + * namespace-local pid to a global one, but main can map the other way without + * trusting anybody: it holds the global pid from SCM_CREDENTIALS at WHOAMI + * time, and the last entry of /proc//status NSpid is that process's pid + * in its own namespace (Linux 4.1+). Recording that as a second key, plus a + * REMOVE_PID variant scoped to one prototype's children, would close it with + * no sender-supplied pid to authenticate. msg->port_msg.pid on the WHOAMI + * message happens to carry the same number, but it is chosen by the sender and + * would need the authentication NSpid makes unnecessary. + */ + +void +nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process) +{ + nxt_int_t ret; + nxt_uint_t level; + nxt_port_t *port; + nxt_runtime_t *rt; + + if (process->state != NXT_PROCESS_STATE_CREATING) { + nxt_port_remove_notify_others(task, process); + + return; + } + + rt = task->thread->runtime; + + /* + * A CREATING child whose initiator is already gone is the expected shape + * of a teardown rather than an anomaly of one, and the record leak noted + * below is moot once the prototype itself is on its way out. + */ + level = nxt_proto_exiting ? NXT_LOG_WARN : NXT_LOG_ALERT; + + if (process->stream != 0) { + port = nxt_runtime_port_find(rt, process->stream_pid, + process->stream_port); + + if (nxt_slow_path(port == NULL)) { + if (rt->is_pid_isolated) { + nxt_log(task, level, "app process (isolated %PI) died before " + "it was created and its start initiator %PI port %d " + "is gone (stream %uD)", process->isolated_pid, + process->stream_pid, (int) process->stream_port, + process->stream); + + } else { + nxt_log(task, level, "app process %PI died before it was " + "created and its start initiator %PI port %d is gone " + "(stream %uD)", process->pid, process->stream_pid, + (int) process->stream_port, process->stream); + } + + /* Nothing can carry the answer; do not leave it armed. */ + process->stream = 0; + + } else { + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, + -1, process->stream, 0, NULL); + + /* + * One answer per start request: nxt_port_rpc_handler() drops a + * stream it no longer knows, but the identifiers come from a + * shared counter and are reused, so a second error for a retired + * stream could land on somebody else's RPC. So ->stream is + * cleared only once the answer has actually been written or + * queued, which is what makes the REMOVE_PID below streamless. + * + * The failure that reaches this call site is an allocation + * failure, and it leaves no message behind. The prototype maps + * no queue for a router port -- nxt_app_new_port_handler() + * closes the queue descriptor every NEW_PORT carries, and main + * maps a queue only for an application port -- so + * nxt_port_socket_write2() never takes the shared-ring path that + * answers NXT_AGAIN, and what is left is + * nxt_port_msg_chk_insert() failing to allocate. Keeping + * ->stream then lets the REMOVE_PID carry it, and + * nxt_router_remove_pid_handler() turns that into the same RPC + * error -- the fallback the CREATED path has always used. + * Clearing it regardless would leave the start RPC armed and + * rebuild the wedge this function exists to close, silently. + * + * That fallback is a second chance, not a guarantee: the + * REMOVE_PID allocates a buffer and a message of its own from + * the same pools and can fail the same way, and then only the + * application's "limits.start_timeout" retires the start. The + * alert below is what makes that case visible instead of silent. + * NXT_OK is not a delivery receipt either, and that shape is + * the one this function cannot cover. When the port is writable + * the message goes out inline; if sendmsg() answers EAGAIN and + * it then cannot be queued for later, it is dropped and only the + * port's error handler is scheduled -- while the caller reads + * NXT_OK. The REMOVE_PID below does not rescue that, and not + * because it is streamless: the EAGAIN cleared the port's + * write_ready, so it is queued rather than sent, and the error + * handler drains the whole queue when it runs. The router is + * told nothing at all. That is the port layer answering for a + * message it dropped, which reaches every RPC reply in the + * daemon rather than this function alone; it is filed as #335, + * and the fix there -- reporting the drop -- is what this + * function's failure branch above already knows how to use. + */ + if (nxt_fast_path(ret == NXT_OK)) { + process->stream = 0; + + } else if (rt->is_pid_isolated) { + nxt_log(task, level, "app process (isolated %PI) died before " + "it was created and could not be reported to its " + "start initiator %PI port %d (stream %uD)", + process->isolated_pid, process->stream_pid, + (int) process->stream_port, process->stream); + + } else { + nxt_log(task, level, "app process %PI died before it was " + "created and could not be reported to its start " + "initiator %PI port %d (stream %uD); the REMOVE_PID " + "that follows carries the report unless it cannot be " + "allocated either", process->pid, process->stream_pid, + (int) process->stream_port, process->stream); + } + } + } + + /* + * Under pid isolation the direct answer is the only vehicle: ->pid is the + * namespace-local one, so no REMOVE_PID may be sent for this child at + * all. When that answer could not be allocated above, the start RPC is + * therefore left armed. What retires it then is the prototype's own + * death -- the router keys that RPC on the process it sent START_PROCESS + * to (nxt_port_rpc_ex_set_peer(), src/nxt_router.c), so main's REMOVE_PID + * for the prototype reaches nxt_port_rpc_remove_peer() -- or, sooner and + * per application, "limits": {"start_timeout"}, which defaults to none. + * Closing it here needs an answer that cannot fail to allocate, which + * this layer has no way to reserve; tracked with the record leak in #310. + */ + + if (nxt_slow_path(rt->is_pid_isolated)) { + nxt_log(task, level, "app process (isolated %PI) died before it was " + "created; it has no globally valid pid to broadcast, so a " + "record the main or router process may hold for it stays " + "until the prototype exits", process->isolated_pid); + + /* No REMOVE_PID will carry it; do not leave it armed. */ + process->stream = 0; + + return; + } + + nxt_port_remove_notify_others(task, process); +} + + static nxt_app_module_t * nxt_app_module_load(nxt_task_t *task, const char *name) { diff --git a/src/nxt_application.h b/src/nxt_application.h index 0b78ea971..4be7ff42f 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -209,4 +209,7 @@ NXT_EXPORT nxt_int_t nxt_unit_default_init(nxt_task_t *task, NXT_EXPORT nxt_int_t nxt_app_set_logs(void); +/* Reports a reaped child of the prototype; see nxt_proto_sigchld_handler(). */ +void nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process); + #endif /* _NXT_APPLICATION_H_INCLIDED_ */ diff --git a/src/nxt_process.h b/src/nxt_process.h index 9575a09d1..102cd6079 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -144,7 +144,17 @@ struct nxt_process_s { const char *name; nxt_port_t *parent_port; + /* + * The start request this process was forked to satisfy: ->stream is the + * initiator's RPC stream, and ->stream_pid/->stream_port address the port + * that RPC is registered on. A creator that can be left holding the only + * armed handler for a child records all three -- the prototype does, in + * nxt_proto_start_process_handler() -- so that a child which dies before + * anything else can report it still gets answered. + */ uint32_t stream; + nxt_pid_t stream_pid; + nxt_port_id_t stream_port; nxt_mp_t *mem_pool; nxt_credential_t *user_cred; diff --git a/src/test/nxt_proto_creating_wedge_test.c b/src/test/nxt_proto_creating_wedge_test.c new file mode 100644 index 000000000..e711c9dd8 --- /dev/null +++ b/src/test/nxt_proto_creating_wedge_test.c @@ -0,0 +1,594 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the CREATING-state start wedge: an application worker + * that died between fork() and PROCESS_CREATED was never reported to anyone, + * so the START_PROCESS RPC it was forked for stayed armed forever and the + * record the main process had already made for it was never retired. + * + * Every child of the prototype exists to satisfy a START_PROCESS request, and + * the router arms an RPC handler for each one. A child that got as far as + * PROCESS_CREATED is reported by REMOVE_PID, which carries ->stream, and + * nxt_router_remove_pid_handler() turns a stream-bearing REMOVE_PID into an + * RPC error. A child still in the CREATING state took neither path: + * nxt_proto_sigchld_handler() skipped the notification for it -- rightly, + * since REMOVE_PID says nothing but a pid and under pid isolation the + * prototype does not yet know a globally valid one -- and nothing else + * answered instead. + * + * The prototype's own death is covered: the router keys that RPC on the + * prototype (nxt_router_start_app_process_handler() calls + * nxt_port_rpc_ex_set_peer(), src/nxt_router.c), so REMOVE_PID for the + * prototype reaches nxt_port_rpc_remove_peer() and fails the start. A child + * that dies while CREATING has no such backstop: the armed handler owns an + * app->pending_processes slot, and a leaked slot makes + * nxt_router_app_can_start() false for good once the application's max is + * reached -- the application never starts another process, and a reload + * recovers only if its own config text changed. + * + * Answering the initiator is not the whole job. A worker that got as far as + * WHOAMI made main create a process record and a port holding main's end of + * the worker's port socket (nxt_main_process_whoami_handler()), and REMOVE_PID + * is the only thing that retires it -- nxt_proc_remove_notify_matrix pairs a + * dying APP with MAIN. Once the start RPC is retired and requests retry, a + * worker that keeps dying in that window costs main one record and one + * descriptor per attempt. So the CREATING case must both answer the initiator + * and send a streamless REMOVE_PID: streamless because the initiator has + * already been answered, and a stream-bearing one would fail the same RPC + * twice. + * + * That REMOVE_PID may only be sent when the pid in it is a usable global key, + * which is what rt->is_pid_isolated says -- the same test nxt_process_create() + * uses to decide whether a forked child may enter the runtime hash at all. + * The pid-isolated case is exercised here too, and it asserts the opposite: + * nothing at all must reach main, because sending a namespace-local pid would + * ask main to remove whatever unrelated process holds that number. + * + * The answer itself can fail to be written -- a full port queue or a failed + * allocation, neither of which leaves anything for the initiator -- and then + * ->stream must survive so that the REMOVE_PID carries it and the router turns + * it into the RPC error the way it always has for a CREATED child. Clearing + * it regardless would rebuild the wedge, silently. + * + * All four cases are driven against one router port and one main port, + * because the point is the split between them. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_PROTO_WEDGE_CREATING_STREAM 0x0BADF00D +#define NXT_PROTO_WEDGE_ISOLATED_STREAM 0x0BADF00E +#define NXT_PROTO_WEDGE_UNSENT_STREAM 0x0BADF00F +#define NXT_PROTO_WEDGE_STUCK_STREAM 0x0BADF010 +#define NXT_PROTO_WEDGE_CREATED_STREAM 0x0C0FFEE1 + + +typedef struct { + nxt_uint_t type; + uint32_t stream; + nxt_pid_t pid; /* REMOVE_PID payload; 0 for a message without one */ +} nxt_proto_wedge_msg_t; + + +/* + * The ports are never write_ready here, so nxt_port_msg_chk_insert() queues + * whatever the handler writes instead of sending it -- which is exactly what + * this test wants to read back. + */ + +static nxt_int_t +nxt_proto_wedge_expect(nxt_thread_t *thr, nxt_port_t *port, + const nxt_proto_wedge_msg_t *expect, nxt_uint_t n, const char *label, + const char *whom) +{ + nxt_pid_t pid; + nxt_uint_t i; + nxt_queue_link_t *link; + nxt_port_send_msg_t *msg; + const nxt_proto_wedge_msg_t *e; + + link = nxt_queue_first(&port->messages); + + for (i = 0; i < n; i++) { + e = &expect[i]; + + if (link == nxt_queue_tail(&port->messages)) { + nxt_log_alert(thr->log, "proto creating wedge test: %s: nothing " + "more was written to the %s, expected message %d of " + "%d (type %d stream %uxD)", label, whom, (int) i + 1, + (int) n, (int) e->type, e->stream); + return NXT_ERROR; + } + + msg = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + + if (msg->port_msg.type != e->type || msg->port_msg.stream != e->stream) + { + nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s got " + "message %d of type %d stream %uxD (expected type %d " + "stream %uxD)", label, whom, (int) i + 1, + (int) msg->port_msg.type, msg->port_msg.stream, + (int) e->type, e->stream); + return NXT_ERROR; + } + + if (e->pid != 0) { + if (msg->buf == NULL + || nxt_buf_used_size(msg->buf) != sizeof(nxt_pid_t)) + { + nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s " + "got a REMOVE_PID carrying no pid", label, whom); + return NXT_ERROR; + } + + nxt_memcpy(&pid, msg->buf->mem.pos, sizeof(nxt_pid_t)); + + if (pid != e->pid) { + nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s " + "was told to remove pid %PI (expected %PI)", + label, whom, pid, e->pid); + return NXT_ERROR; + } + } + + link = nxt_queue_next(link); + } + + if (link != nxt_queue_tail(&port->messages)) { + msg = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + + nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s got an " + "extra message of type %d stream %uxD after the %d " + "expected", label, whom, (int) msg->port_msg.type, + msg->port_msg.stream, (int) n); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static void +nxt_proto_wedge_drain(nxt_task_t *task, nxt_port_t *port) +{ + /* + * Queued messages hold a port reference each; nxt_port_test_run_error + * _handler() is how the other port tests give them back, and it is what + * lets the pool cleanup assertions run under a debug build. + */ + nxt_port_test_run_error_handler(task, port); +} + + +nxt_int_t +nxt_proto_creating_wedge_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *router_port, *main_port, *app_port; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *process; + nxt_event_engine_t engine, *saved_engine; + nxt_proto_wedge_msg_t expect[2]; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "proto creating wedge test started"); + + ret = NXT_ERROR; + app_port = NULL; + main_port = NULL; + router_port = NULL; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * nxt_port_remove_notify_others() allocates its payload from + * task->thread->engine->mem_pool; only the members the reached code + * touches are set, as in src/test/nxt_router_new_port_test.c. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* + * The port the start request came from. Its pid must differ from + * nxt_pid: nxt_port_remove_notify_others() skips the sender's own + * process, so a router sharing this process's pid would make the + * CREATED case pass for the wrong reason. + */ + + router_port = nxt_runtime_process_port_create(task, rt, nxt_pid + 1, 0, + NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + router_port->pair[0] = -1; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The process that answered the worker's WHOAMI and is holding a record + * and a descriptor for it. nxt_proc_remove_notify_matrix pairs a dying + * APP with MAIN, so this is the port the streamless REMOVE_PID has to + * reach. + */ + + main_port = nxt_runtime_process_port_create(task, rt, nxt_pid + 3, 0, + NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + /* The child being reaped. Deliberately not in rt->processes: a + CREATING child under pid isolation is kept out of the global pid hash, + and the CREATED case does not need to find itself there either. */ + + process = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(process == NULL)) { + goto done; + } + + process->pid = nxt_pid + 2; + process->isolated_pid = nxt_pid + 2; + nxt_queue_init(&process->ports); + nxt_queue_init(&process->children); + + /* nxt_process_type() reads the first port, and the notify matrix pairs + NXT_PROCESS_APP with NXT_PROCESS_ROUTER and NXT_PROCESS_MAIN. */ + + app_port = nxt_port_new(task, 0, process->pid, NXT_PROCESS_APP); + if (nxt_slow_path(app_port == NULL)) { + goto done; + } + + app_port->pair[0] = -1; + app_port->pair[1] = -1; + app_port->socket.fd = -1; + + nxt_queue_insert_tail(&process->ports, &app_port->link); + + /* + * A child that died before PROCESS_CREATED, with the prototype in main's + * pid namespace. The old code left this silent, so the initiator's RPC + * stayed armed for the router's lifetime and main kept the record and the + * descriptor it made at WHOAMI time until the prototype exited. + */ + + process->state = NXT_PROCESS_STATE_CREATING; + process->stream = NXT_PROTO_WEDGE_CREATING_STREAM; + process->stream_pid = router_port->pid; + process->stream_port = router_port->id; + + nxt_proto_child_exited(task, process); + + /* + * The initiator is answered directly, and only then told to drop the pid + * -- streamless, or nxt_router_remove_pid_handler() would fail the same + * RPC a second time. + */ + + expect[0].type = _NXT_PORT_MSG_RPC_ERROR; + expect[0].stream = NXT_PROTO_WEDGE_CREATING_STREAM; + expect[0].pid = 0; + expect[1].type = _NXT_PORT_MSG_REMOVE_PID; + expect[1].stream = 0; + expect[1].pid = process->pid; + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 2, + "a child that died while creating", + "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_proto_wedge_expect(thr, main_port, &expect[1], 1, + "a child that died while creating", + "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * One answer per start request. Stream identifiers come from a shared + * counter and are reused, so a second error for a stream already retired + * could land on an unrelated RPC. + */ + + if (nxt_slow_path(process->stream != 0)) { + nxt_log_alert(thr->log, "proto creating wedge test: the answered " + "start request is still armed on the process"); + ret = NXT_ERROR; + goto done; + } + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + /* + * The same child under pid isolation. ->pid is then the namespace-local + * pid fork() returned, and main keyed its record on the global pid it + * read from SCM_CREDENTIALS, so there is no pid to send: the initiator is + * still answered, and nothing at all may reach main. Sending the + * namespace-local pid would ask main to remove whatever unrelated process + * happens to hold that number, which is worse than the leak this closes. + */ + + rt->is_pid_isolated = 1; + + process->state = NXT_PROCESS_STATE_CREATING; + process->stream = NXT_PROTO_WEDGE_ISOLATED_STREAM; + + nxt_proto_child_exited(task, process); + + expect[0].type = _NXT_PORT_MSG_RPC_ERROR; + expect[0].stream = NXT_PROTO_WEDGE_ISOLATED_STREAM; + expect[0].pid = 0; + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 1, + "a pid-isolated child that died while " + "creating", "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 0, + "a pid-isolated child that died while " + "creating", "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + rt->is_pid_isolated = 0; + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + /* + * The same child, but the RPC error cannot be written. The prototype + * maps no queue for a router port, so nxt_port_socket_write() answers + * NXT_ERROR here -- nxt_port_msg_chk_insert() failing to allocate -- and + * leaves nothing behind for the initiator. Clearing ->stream regardless + * would send a streamless REMOVE_PID and restore the wedge with no + * diagnostic at all. The stream must survive the failed write instead, + * so that the REMOVE_PID carries it and nxt_router_remove_pid_handler() + * turns it into the RPC error -- the fallback the CREATED case below has + * always relied on. Only the allocation failure can be injected here; it + * exercises the same discarded return. + */ + + process->state = NXT_PROCESS_STATE_CREATING; + process->stream = NXT_PROTO_WEDGE_UNSENT_STREAM; + + nxt_port_test_msg_alloc_failures(1); + + nxt_proto_child_exited(task, process); + + nxt_port_test_msg_alloc_failures(0); + + expect[0].type = _NXT_PORT_MSG_REMOVE_PID; + expect[0].stream = NXT_PROTO_WEDGE_UNSENT_STREAM; + expect[0].pid = process->pid; + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 1, + "a child whose start error could not be " + "written", "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 1, + "a child whose start error could not be " + "written", "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + /* + * Both at once: pid-isolated, and the RPC error cannot be written. This + * is the one combination nothing can recover locally -- the direct answer + * is the only vehicle under isolation, and the REMOVE_PID that covers the + * failed write otherwise cannot be sent, because ->pid is the + * namespace-local one. + * + * What must not happen is a repair that looks like one: broadcasting that + * pid anyway would ask main and the router to remove whatever unrelated + * process holds the number, and a prototype's namespace-local counter + * climbs straight into the range the daemon's own pids occupy. So the + * assertion is an absence on both ports, and it is what turns red if a + * later change decides to send something here. + * + * The start RPC is left armed, which the code says out loud. It is + * retired when the prototype itself dies -- the router keys that RPC on + * the prototype (nxt_port_rpc_ex_set_peer(), src/nxt_router.c), so the + * REMOVE_PID main sends for it reaches nxt_port_rpc_remove_peer() -- + * or sooner by "limits": {"start_timeout"} where the application sets + * one. ->stream is still cleared, since nothing may carry it later. + */ + + rt->is_pid_isolated = 1; + + process->state = NXT_PROCESS_STATE_CREATING; + process->stream = NXT_PROTO_WEDGE_STUCK_STREAM; + + nxt_port_test_msg_alloc_failures(1); + + nxt_proto_child_exited(task, process); + + nxt_port_test_msg_alloc_failures(0); + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 0, + "a pid-isolated child whose start error " + "could not be written", "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 0, + "a pid-isolated child whose start error " + "could not be written", "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(process->stream != 0)) { + nxt_log_alert(thr->log, "proto creating wedge test: a pid-isolated " + "child whose start error could not be written left the " + "start request armed on the process"); + ret = NXT_ERROR; + goto done; + } + + rt->is_pid_isolated = 0; + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + /* + * A child that reached PROCESS_CREATED keeps the path it always had: + * REMOVE_PID carrying the stream, which the router turns into an RPC + * error of its own, and which retires main's record as it always did. + */ + + process->state = NXT_PROCESS_STATE_CREATED; + process->stream = NXT_PROTO_WEDGE_CREATED_STREAM; + + nxt_proto_child_exited(task, process); + + expect[0].type = _NXT_PORT_MSG_REMOVE_PID; + expect[0].stream = NXT_PROTO_WEDGE_CREATED_STREAM; + expect[0].pid = process->pid; + + ret = nxt_proto_wedge_expect(thr, router_port, expect, 1, + "a child that died after it was created", + "start initiator"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 1, + "a child that died after it was created", + "main process"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + nxt_proto_wedge_drain(task, router_port); + nxt_proto_wedge_drain(task, main_port); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "proto creating wedge test passed"); + + ret = NXT_OK; + +done: + + /* + * Drain first, then hand each port back the way its owner would: these + * two were made by nxt_runtime_process_port_create() and carry pools of + * their own, which the fixture's pool does not cover. Through the + * ownership path rather than by destroying those pools directly -- + * nxt_port_mp_cleanup() asserts the descriptors are gone and the use + * count is zero, so nxt_mp_destroy() here would abort a debug build while + * looking clean in a release one. nxt_port_close() is a no-op on the + * -1 descriptors the fixture set; nxt_runtime_port_remove() drops the + * rt->ports reference, which is the last one, and frees the port. + */ + + if (router_port != NULL) { + nxt_proto_wedge_drain(task, router_port); + + if (!nxt_queue_is_empty(&router_port->messages)) { + nxt_log_alert(thr->log, "proto creating wedge test: the router " + "port still holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_close(task, router_port); + nxt_runtime_port_remove(task, router_port); + } + + if (main_port != NULL) { + nxt_proto_wedge_drain(task, main_port); + + if (!nxt_queue_is_empty(&main_port->messages)) { + nxt_log_alert(thr->log, "proto creating wedge test: the main " + "port still holds queued messages after teardown"); + ret = NXT_ERROR; + } + + nxt_port_close(task, main_port); + nxt_runtime_port_remove(task, main_port); + } + + if (app_port != NULL) { + nxt_queue_remove(&app_port->link); + + /* + * nxt_queue_remove() nulls the link only in a debug build + * (src/nxt_queue.h:130-148). nxt_port_release() reads a non-NULL + * link.next as "this port belongs to a process" and dereferences + * port->process, which nxt_port_new() never set -- guarded by an + * nxt_assert() that compiles out of a release build, so the test + * would segfault in teardown there and nowhere else. + */ + app_port->link.next = NULL; + app_port->link.prev = NULL; + + nxt_port_use(task, app_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 6bd1c5b8e..98f1cbd21 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -241,6 +241,10 @@ main(int argc, char **argv) return 1; } + if (nxt_proto_creating_wedge_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 78866e78f..54a0326b7 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -84,6 +84,7 @@ nxt_int_t nxt_router_start_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); +nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); From c8d401863c02d4854bc341c749a755ae637fa191 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 12 Sep 2026 22:33:17 +0200 Subject: [PATCH 136/391] docs(changes): record the START_PROCESS reply fix 3e3ce864 ("main: reply to every START_PROCESS that cannot start a process") landed with no CHANGES entry. 1.36.2 is not released yet, so write one next to the fix for the other half of the same wedge. Co-Authored-By: Claude Opus 5 --- CHANGES | 7 +++++++ docs/changes.xml | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/CHANGES b/CHANGES index 9e7845d69..0915dc3e2 100644 --- a/CHANGES +++ b/CHANGES @@ -23,6 +23,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 the "credential" namespace, and applications run as unitd's own uid. Every other capget() failure remains fatal. + *) Bugfix: the main process did not answer a START_PROCESS request it + could not carry out -- an unknown router port, a sender that is not + the router, or an allocation that failed -- so the start stayed + pending. The application then parked every later start on a + prototype that was not coming, and the requests waiting on it were + never failed. + *) Bugfix: an application worker that died before it finished starting was reported to nobody, so the start it was forked for stayed pending forever. Once the application reached its "processes" maximum it diff --git a/docs/changes.xml b/docs/changes.xml index 344743ab6..0e5bce7cd 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -44,6 +44,16 @@ remains fatal. + + +the main process did not answer a START_PROCESS request it could not carry +out -- an unknown router port, a sender that is not the router, or an +allocation that failed -- so the start stayed pending. The application then +parked every later start on a prototype that was not coming, and the +requests waiting on it were never failed. + + + an application worker that died before it finished starting was reported to From 5162eee8ac49b60ea6e23f67120d2f0289c49c5c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 9 Sep 2026 07:37:02 +0200 Subject: [PATCH 137/391] unitctl: keep working against a server whose config it cannot decode nxt_conf_json_parse_string() keeps every byte from 0x20 upwards that it finds inside a JSON string, so a configuration stored by any released version -- 1.34 LTS and every 1.36.x -- can hold bytes that are not valid UTF-8, and GET /config hands them back verbatim. serde_json rejects such a body whole, so unitctl failed on exactly the configuration it is most needed for: "unitctl export" and every "unitctl execute" against /config died with "JSON error [path=/config]", naming neither the reason nor the member at fault. Decode response bodies through json_body::decode(), which tries the strict decode first -- a UTF-8 body takes the path it always took, byte for byte -- and only falls back to a replacing decode when the bytes themselves are not UTF-8. The fallback is never silent: it names the RFC 6901 pointer of every member it altered, and says the value shown is not the value stored. A body that is valid UTF-8 but malformed, or that does not match the type asked for, still fails. Altered, and not merely holding a U+FFFD. EF BF BD is valid UTF-8, so a U+FFFD an operator stored arrives intact and reads exactly like one this decode wrote -- and naming it would send them to "unitctl edit"'s recovery advice for a member that was never damaged, when saying where the damage is is the whole point. So the bytes are decoded a second time with the undecodable ones written as ASCII, and a string is reported when its two decodings differ: a stored U+FFFD reads the same both times. The two trees are walked together, by position, which the member order and the refusal above make safe; nothing searches the second decode for a marker, because a value that is the marker text -- or JSON's escaping spelling it out, which no check of the raw bytes would see -- would defeat that. The second decode's objects keep every member, duplicate names included, rather than being a map: the marker is ordinary text, so a document can hold a member named it beside one whose name decodes to it, and a map would keep one of the two and leave the sides disagreeing on length. If the second decode cannot be made or the sides still do not line up, every U+FFFD is named as before: a comparison that could not be made is not grounds for saying less. Which member was altered decides what can be done about it. A value can be repaired with a PUT to the path shown; a *name* cannot, because that path is not one the server has -- it shows U+FFFD where the server holds a byte, so a PUT to it would address some other member, and percent- encoding the U+FFFD only picks a different key again. Fidelity reports the two apart, and "unitctl edit" gives each case its own advice rather than one instruction and then an exception to it: the values it lists are repaired with a PUT to the path shown, and for the names it says no PUT repairs them, that the name as stored comes from "unitctl export", which keeps the server's bytes verbatim, and that what to rewrite is the nearest parent that does decode. The comparison needs the server's member order, because it pairs the two decodes by position. serde_json keeps that order only with its preserve_order feature, and unit-client-rs asked for it through a default feature of its own -- so "cargo build --no-default-features" produced a sorted map, paired the two sides wrongly, and reported members that were never touched. The feature is now on the dependency itself, where nothing can switch it off, and the crate's own feature table is gone: it existed only to carry this, and an empty one would invite it back. A test pins member order so that removing the feature fails the suite rather than changing the answers quietly. A member name is whatever the server stored, and it reaches the terminal through that warning. serde_json decodes the escape for ESC into a real ESC, and accepts an escaped newline too, so a name can carry an ANSI sequence the terminal would act on or forge a line of unitctl's own output. Every control character in a pointer is written out as the escape JSON would use, and so is every one in the four strings taken out of a JSON error body -- the error, the detail, the location and the suggestion -- and in a body shown because it would not parse. Written out, and not replaced with U+FFFD: that character is what the warning beside it says is a byte the server stores and UTF-8 cannot carry, and a control character in a name is valid UTF-8 that arrived intact, so showing it as U+FFFD would send the operator hunting a byte that does not exist. A body whose member names differ only in those bytes fails as well. Every undecodable byte becomes the same character, so "app-\xFF" and "app-\xFE" arrive as one name twice; serde_json keeps the last of two equal names, and a whole application would be missing from what is shown, under a warning that mentions only replaced bytes. The point of the fallback is to show the operator what is there, so a document known to be incomplete is not shown at all. The replacing decode runs through a wrapper that refuses a name it has already seen, at every depth, which asks the question of the names the parser decoded: any test of how the bytes are spelled in the JSON source can be forged by JSON's own escapes. It refuses every repeated name, so it says two members share a name and offers undecodable bytes as the usual reason rather than asserting a cause it did not check. A configuration is also a JSON object, and the type is what says so. "unitctl export" and "unitctl edit" read /config as UnitSerializableMap, which is what the deserializing helper they called before returned, so a success status carrying "null", an array or a bare string is refused rather than archived as a backup or opened in the editor and handed back as the whole configuration. Writing back is the other half. A Rust string cannot carry those bytes, so anything routed through serde_json on the way out returns something other than what it was shown. "unitctl export" now stores the bytes the server sent, verbatim, when a Rust string cannot carry them, and says that what it wrote is a file "unitctl import" cannot replay. Being unreadable as UTF-8 is not on its own a reason to store a body: both ways of reading it ask for a configuration first, so a truncated answer or an intermediary's page carrying a success status still fails instead of being archived as one. "unitctl edit" refuses rather than rewrite a configuration the user only meant to open, pointing at the member and at the read-only way to inspect it. While here, two adjacent defects the same code path exposed: - UnitSerializableMap was a HashMap, which does not keep member order. /config came back with its top-level members in a different order on every single run, so two "unitctl export"s of one unchanged configuration disagreed and "unitctl edit" opened a differently shuffled document each time. serde_json::Map with the crate's preserve_order feature keeps the server's order. hjson input was shuffled by a HashMap of its own on the way in; nu_json::Map, which the same feature makes a LinkedHashMap, keeps that order too. - A response body that would not parse as JSON was replaced by a bare "JSON error", hiding the text -- an intermediary's 502 page, say -- at the moment it matters most. Show the body instead, and let JsonError name its cause when it is raised. A success status is no guarantee that Unit answered, so the body is shown there too: a proxy or a captive portal that returns 200 with a page of its own is the ordinary way that arm is reached, and "unitctl export" and "unitctl edit" say the same thing when they refuse such a page. That body is neither bounded nor trusted, so at most 2 KiB of it is shown, with its control characters written out. Two limits of this change, stated because the title is broader than what it delivers. "unitctl status" and "unitctl listeners" still fail on such a body: they decode through the generated OpenAPI client rather than this path, and answer "invalid unicode code point" as before. And UnitSerializableMap is a public alias of unit-client-rs, so moving it from HashMap to serde_json::Map is a breaking change for anything outside this tree that names the type, as is dropping that crate's "default" feature. Three lines are reformatted that this change does not otherwise touch: describe_error_body()'s "let location = place.map(...)" and the two copies of "if cli.control_socket_addresses.as_ref().map_or(...)". They are byte-identical on master and rustfmt disagreed with them there too, so this is not churn from the change -- it is what makes "cargo fmt --check" clean on the files the change already edits, rather than a standing false signal. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/unit-client-rs/Cargo.toml | 9 +- tools/unitctl/unit-client-rs/src/json_body.rs | 1358 +++++++++++++++++ tools/unitctl/unit-client-rs/src/lib.rs | 1 + .../unitctl/unit-client-rs/src/unit_client.rs | 204 ++- tools/unitctl/unitctl/src/cmd/edit.rs | 43 +- tools/unitctl/unitctl/src/cmd/save.rs | 211 ++- tools/unitctl/unitctl/src/inputfile.rs | 115 +- tools/unitctl/unitctl/src/requests.rs | 30 +- tools/unitctl/unitctl/src/unitctl_error.rs | 31 + 9 files changed, 1955 insertions(+), 47 deletions(-) create mode 100644 tools/unitctl/unit-client-rs/src/json_body.rs diff --git a/tools/unitctl/unit-client-rs/Cargo.toml b/tools/unitctl/unit-client-rs/Cargo.toml index 834c9b59d..a430dac2e 100644 --- a/tools/unitctl/unit-client-rs/Cargo.toml +++ b/tools/unitctl/unit-client-rs/Cargo.toml @@ -8,10 +8,6 @@ license = "Apache-2.0" [lib] name = "unit_client_rs" -[features] -# this preserves the ordering of json -default = ["serde_json/preserve_order"] - [dependencies] custom_error = "1.9" hyper = { version = "1", features = ["http1", "http2"] } @@ -21,7 +17,10 @@ hyperlocal = "0.9" http-body-util = "0.1" bytes = "1" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +# preserve_order is required, not optional. json_body compares two decodes of +# the same body member by member, in order, and a sorted map would pair them +# wrongly. It also keeps the server's member order in what unitctl prints. +serde_json = { version = "1.0", features = ["preserve_order"] } sysinfo = "0.39" tokio = { version = "1.34", features = ["macros"] } futures = "0.3" diff --git a/tools/unitctl/unit-client-rs/src/json_body.rs b/tools/unitctl/unit-client-rs/src/json_body.rs new file mode 100644 index 000000000..4ff2eb9a0 --- /dev/null +++ b/tools/unitctl/unit-client-rs/src/json_body.rs @@ -0,0 +1,1358 @@ +//! Decoding control-API response bodies that are not guaranteed to be UTF-8. +//! +//! `nxt_conf_json_parse_string()` keeps every byte from 0x20 upwards that it +//! finds inside a JSON string, so a configuration stored by any released +//! version -- 1.34 LTS and every 1.36.x -- can hold bytes that are not valid +//! UTF-8, and `GET /config` hands them back verbatim. `serde_json` rejects +//! such a body whole, which would leave unitctl unable to look at the one +//! configuration it is most needed for. +//! +//! Everything here is a fallback. A body that is valid UTF-8 takes the path +//! it always took, byte for byte; only a body that cannot be decoded strictly +//! is decoded again with the undecodable bytes replaced, and that never +//! happens silently. + +use serde::de::{DeserializeOwned, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use serde_json::Value; +use std::fmt; + +/// The character `String::from_utf8_lossy` leaves behind. +const REPLACEMENT: char = '\u{fffd}'; + +/// How many members a warning names before it stops listing them. +const MAX_LISTED: usize = 8; + +/// How much of the server's answer survived decoding. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum Fidelity { + /// The body was valid UTF-8: the value is exactly what the server sent. + #[default] + Exact, + /// The body was not valid UTF-8. Every byte that could not be decoded was + /// replaced with U+FFFD, so the value is readable but is *not* what the + /// server stores. `members` holds the RFC 6901 pointers of the members + /// this altered, and `names` the ones among them whose *name* was altered + /// rather than its value -- a pointer in `names` cannot be PUT to, because + /// the path itself holds U+FFFD where the server holds a byte. + Replaced { members: Vec, names: Vec }, +} + +impl Fidelity { + pub fn is_exact(&self) -> bool { + matches!(self, Fidelity::Exact) + } + + /// The affected members, as a comma-separated list of RFC 6901 pointers. + pub fn members(&self) -> String { + let members = match self { + Fidelity::Exact => return String::new(), + Fidelity::Replaced { members, .. } => members, + }; + + if members.is_empty() { + // The bytes did not survive as a string value, so there is nothing + // to point at. Say that rather than printing an empty list. + return "an unlocatable part of the document".to_string(); + } + + list(members) + } + + /// The affected members whose *name* was altered, as a comma-separated list + /// of RFC 6901 pointers, or empty when none were. + /// + /// These are the ones no PUT can repair: the pointer holds U+FFFD where the + /// server holds a byte, so the path names a member that does not exist. + pub fn replaced_names(&self) -> String { + match self { + Fidelity::Exact => String::new(), + Fidelity::Replaced { names, .. } => list(names), + } + } + + /// The affected members whose *value* was altered, as a comma-separated list + /// of RFC 6901 pointers, or empty when none were. + /// + /// These are the ones a PUT to the path shown does repair. + pub fn replaced_values(&self) -> String { + let (members, names) = match self { + Fidelity::Exact => return String::new(), + Fidelity::Replaced { members, names } => (members, names), + }; + + let values: Vec = members + .iter() + .filter(|member| !names.contains(member)) + .cloned() + .collect(); + + list(&values) + } + + /// The warning to print when a body had to be decoded lossily, or `None` + /// when it did not. + pub fn warning(&self, path: &str) -> Option { + if self.is_exact() { + return None; + } + + Some(format!( + "Warning: {} returned bytes that are not valid UTF-8.\n\ + Warning: they are shown as U+FFFD and are NOT what the server stores: {}", + path, + self.members() + )) + } +} + +/// How much of a response body that would not parse is shown. +const MAX_SHOWN_BODY: usize = 2048; + +/// Pointers as one line for a person to read: capped, and with the control +/// characters a name can hold written out. +fn list(members: &[String]) -> String { + let listed = members + .iter() + .take(MAX_LISTED) + .map(|member| { + if member.is_empty() { + "the whole document".to_string() + } else { + one_line(member) + } + }) + .collect::>() + .join(", "); + + match members.len().checked_sub(MAX_LISTED) { + Some(rest) if rest > 0 => format!("{} and {} more", listed, rest), + _ => listed, + } +} + +/// How a control character is written when text from the server is printed. +/// +/// Deliberately not U+FFFD. That character is what [`Fidelity::warning`] tells +/// the operator is a byte the server stores and UTF-8 cannot carry, and a +/// control character in a name is not one: it is valid UTF-8 that arrived +/// intact, so showing it as U+FFFD would send the operator hunting a byte that +/// does not exist -- and `unitctl_error.rs`'s advice on such a member would be +/// wrong. The escape JSON itself would use keeps the two apart and still says +/// what was really there, which a `?` would throw away. +fn escaped(character: char) -> String { + format!("\\u{:04x}", character as u32) +} + +/// One line of text from the server, made safe to print. +/// +/// A member name can hold any character JSON can spell, and `serde_json` decodes +/// the escape for ESC into a real ESC: a name can carry an ANSI escape sequence +/// that the terminal would act on, or a newline or carriage return, which would +/// let it forge a line of unitctl's own output. Only a raw control *byte* is +/// refused by the parser, and a name reaches stderr through every warning about +/// replaced bytes, so every control character is written out by [`escaped`]. +/// +/// Every one of them, newline and tab included, because a pointer is one line. +/// [`body_for_display`] keeps those two, because a page is many lines. +pub(crate) fn one_line(text: &str) -> String { + let mut shown = String::with_capacity(text.len()); + + for character in text.chars() { + if character.is_control() { + shown.push_str(&escaped(character)); + } else { + shown.push(character); + } + } + + shown +} + +/// Render a response body that would not parse, for a terminal. +/// +/// Whatever answered the request wrote this: an intermediary's page, a truncated +/// answer, a captive portal's sign-in form. It is neither bounded -- an error +/// page can be megabytes -- nor trusted, and the control characters in it are +/// the ones a terminal acts on, so an escape sequence in a 502 page would +/// otherwise reach the user's screen. +/// +/// Newline and tab are kept, because a page is many lines. Every other control +/// character is written out by [`escaped`], and the U+FFFD that can appear here +/// is the real thing: this decodes lossily, so those are bytes that could not be +/// decoded. +pub fn body_for_display(bytes: &[u8]) -> String { + // Rendering never shortens: a character of n bytes comes back as n bytes, or + // as the 3 of U+FFFD, or as the 6 of an escape. So this many input bytes + // always suffice to fill the budget, and stopping there is what keeps a + // multi-megabyte page from being copied whole to show 2 KiB of it. The few + // extra bytes cover a character the cut lands inside. + let head = &bytes[..bytes.len().min(MAX_SHOWN_BODY + 4)]; + let mut shown = String::new(); + + for character in String::from_utf8_lossy(head).chars() { + let keep = match character { + '\n' | '\t' => character.to_string(), + control if control.is_control() => escaped(control), + other => other.to_string(), + }; + + if shown.len() + keep.len() > MAX_SHOWN_BODY { + // Said of the output, because that is what was measured: a page of + // control or undecodable bytes renders to several bytes per byte, so + // this is not a count of the body. + shown.push_str(&format!("\n[... truncated after {} bytes of output]", MAX_SHOWN_BODY)); + return shown; + } + + shown.push_str(&keep); + } + + shown +} + +/// Deserialize a control-API response body, tolerating bytes that are not +/// valid UTF-8. +/// +/// The strict decode is tried first and is what almost every body takes, so a +/// server that answers in UTF-8 is unaffected. Only when the bytes themselves +/// are not UTF-8 is the body decoded again with the undecodable bytes +/// replaced; a body that is valid UTF-8 but malformed JSON, or that does not +/// match `T`, still fails with the error it always failed with. +/// +/// A replacing decode that would lose a member fails instead. See +/// [`Unambiguous`]: the point of the fallback is to show the operator what is +/// there, so showing less than what is there without saying so is the one +/// outcome it must not have. +pub fn decode(bytes: &[u8]) -> Result<(T, Fidelity), serde_json::Error> { + match serde_json::from_slice::(bytes) { + Ok(value) => Ok((value, Fidelity::Exact)), + Err(strict) => { + if std::str::from_utf8(bytes).is_ok() { + return Err(strict); + } + + let text = String::from_utf8_lossy(bytes); + let Unambiguous(value) = serde_json::from_str(&text)?; + + // A second decode, so that a U+FFFD the operator stored is not + // reported as one this decode wrote. + let marked = serde_json::from_str::(&marked_decode(bytes)).ok(); + let altered = replaced_members(&value, marked.as_ref()); + let typed = serde_json::from_value(value)?; + + Ok(( + typed, + Fidelity::Replaced { + members: altered.all, + names: altered.names, + }, + )) + } + } +} + +/// What `decode` says when it will not show a document it knows is incomplete. +const AMBIGUOUS_NAME: &str = "two members of one object share a name, so one of them would be \ + dropped; the usual reason is names that differ only in bytes that are \ + not valid UTF-8, which decode alike. unitctl will not show a document \ + it knows is incomplete: read these bytes with 'unitctl export' or curl"; + +/// A JSON document whose member names are known to be distinct. +/// +/// `String::from_utf8_lossy` maps every undecodable byte to the same character, +/// so `"app-\xFF"` and `"app-\xFE"` arrive as one name twice. A +/// `serde_json::Map` keeps the last of two equal names, so the other member's +/// whole value -- an application, a listener -- would be missing from what is +/// shown, and the warning about replaced bytes says nothing about the loss. +/// Deserializing through this refuses instead, at every depth. +/// +/// The question is asked of the names the parser decoded, which is where the +/// answer lives. Asking it of any rewriting of the JSON *source* cannot work, +/// because JSON's own escapes forge whatever the rewriting uses: a `%` written +/// as the escape `\u0025` is not a `%` in the source and is one after parsing. +struct Unambiguous(Value); + +impl<'de> Deserialize<'de> for Unambiguous { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(UnambiguousVisitor) + } +} + +struct UnambiguousVisitor; + +impl<'de> Visitor<'de> for UnambiguousVisitor { + type Value = Unambiguous; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("any JSON value") + } + + fn visit_map>(self, mut access: A) -> Result { + let mut object = serde_json::Map::new(); + + while let Some(name) = access.next_key::()? { + // Recursing through the same wrapper is what makes this hold inside + // nested objects and arrays, not only at the top level. + let Unambiguous(member) = access.next_value()?; + + if object.insert(name, member).is_some() { + return Err(serde::de::Error::custom(AMBIGUOUS_NAME)); + } + } + + Ok(Unambiguous(Value::Object(object))) + } + + fn visit_seq>(self, mut access: A) -> Result { + let mut items = Vec::new(); + + while let Some(Unambiguous(item)) = access.next_element()? { + items.push(item); + } + + Ok(Unambiguous(Value::Array(items))) + } + + fn visit_bool(self, value: bool) -> Result { + Ok(Unambiguous(Value::Bool(value))) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(Unambiguous(Value::from(value))) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(Unambiguous(Value::from(value))) + } + + fn visit_f64(self, value: f64) -> Result { + Ok(Unambiguous(Value::from(value))) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Unambiguous(Value::String(value.to_owned()))) + } + + fn visit_string(self, value: String) -> Result { + Ok(Unambiguous(Value::String(value))) + } + + fn visit_unit(self) -> Result { + Ok(Unambiguous(Value::Null)) + } + + fn visit_none(self) -> Result { + Ok(Unambiguous(Value::Null)) + } + + fn visit_some>(self, deserializer: D) -> Result { + Deserialize::deserialize(deserializer) + } +} + +/// What the second decode writes where bytes could not be decoded. +/// +/// Nothing ever searches for this text, and nothing about it needs to be +/// special: a string is reported because its two decodings *differ*, not because +/// it contains a marker. The only requirements are that it is not U+FFFD, so +/// that a byte which could not be decoded reads differently in the two +/// decodings, and that it is plain ASCII, so that decoding cannot produce it. +/// +/// Searching for it instead would be forgeable, which is why this does not: a +/// value that is literally this text defeats a search, and so does JSON's own +/// escaping spelling it out -- and checking that the text is absent from the raw +/// bytes does not see the escaped spelling, so that check would not help either. +const MARKER: &str = "?"; + +/// The same bytes decoded a second time, with every subsequence that is not +/// valid UTF-8 written as [`MARKER`] instead of U+FFFD. +/// +/// Grouped as `String::from_utf8_lossy` groups: one replacement per maximal +/// ill-formed subsequence. So the two decodings of one string agree on how many +/// replacements it has and where they fall, and two names equal under one are +/// equal under the other -- which is what lets the two trees be walked together. +fn marked_decode(bytes: &[u8]) -> String { + let mut text = String::new(); + let mut rest = bytes; + + loop { + let error = match std::str::from_utf8(rest) { + Ok(valid) => { + text.push_str(valid); + return text; + } + Err(error) => error, + }; + + let (valid, invalid) = rest.split_at(error.valid_up_to()); + text.push_str(std::str::from_utf8(valid).unwrap_or_default()); + text.push_str(MARKER); + + // `error_len()` is None when the bytes end mid-sequence: the rest of the + // input is that one incomplete sequence. + let undecodable = error.error_len().unwrap_or(invalid.len()); + rest = &invalid[undecodable..]; + } +} + +/// The second decode's tree, keeping every member of every object, in order. +/// +/// Deliberately not a `serde_json::Map`. The marker is ordinary text, so a +/// document can hold a member whose name *is* the marker beside one whose name +/// decodes to it -- a literal `"?"` key and a key holding a byte that cannot be +/// decoded -- and a map would keep one of the two. The two sides would then +/// disagree on length, the comparison would fall back for that whole object, and +/// every U+FFFD in it would be named: exactly the false positive the comparison +/// exists to prevent, reachable by writing the marker in the document. +/// +/// This is the third in-band token this check has had, and the first two were +/// each defeated by content that could contain them. Keeping the pairs ends +/// that: the marker's text stops mattering to the *structure*, so no document +/// can make the two sides fail to line up. +#[derive(Debug)] +enum Marked { + Object(Vec<(String, Marked)>), + Array(Vec), + String(String), + /// A number, a boolean or null: nothing that can hold a replaced byte. + Scalar, +} + +impl<'de> Deserialize<'de> for Marked { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(MarkedVisitor) + } +} + +struct MarkedVisitor; + +impl<'de> Visitor<'de> for MarkedVisitor { + type Value = Marked; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("any JSON value") + } + + fn visit_map>(self, mut access: A) -> Result { + let mut members = Vec::new(); + + while let Some(name) = access.next_key::()? { + members.push((name, access.next_value()?)); + } + + Ok(Marked::Object(members)) + } + + fn visit_seq>(self, mut access: A) -> Result { + let mut items = Vec::new(); + + while let Some(item) = access.next_element()? { + items.push(item); + } + + Ok(Marked::Array(items)) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Marked::String(value.to_owned())) + } + + fn visit_string(self, value: String) -> Result { + Ok(Marked::String(value)) + } + + fn visit_bool(self, _value: bool) -> Result { + Ok(Marked::Scalar) + } + + fn visit_i64(self, _value: i64) -> Result { + Ok(Marked::Scalar) + } + + fn visit_u64(self, _value: u64) -> Result { + Ok(Marked::Scalar) + } + + fn visit_f64(self, _value: f64) -> Result { + Ok(Marked::Scalar) + } + + fn visit_unit(self) -> Result { + Ok(Marked::Scalar) + } + + fn visit_none(self) -> Result { + Ok(Marked::Scalar) + } + + fn visit_some>(self, deserializer: D) -> Result { + Deserialize::deserialize(deserializer) + } +} + +/// Where a replacing decode altered the document. +/// +/// `values` and `names` are RFC 6901 pointers. They are kept apart because the +/// advice differs: a pointer whose *value* was altered can be PUT to, and one +/// whose *name* was altered cannot -- the path itself holds U+FFFD where the +/// server holds a byte, so a PUT would address a different member. +#[derive(Debug, Default)] +struct Altered { + all: Vec, + names: Vec, +} + +/// The members this decode actually altered, and which of them are member names. +/// +/// Not every U+FFFD in the decoded tree is one of ours. `EF BF BD` is valid +/// UTF-8, so a U+FFFD an operator stored arrives intact -- and naming it would +/// send them to `edit`'s recovery advice for a member that was never damaged, +/// while the point of this warning is to say where the damage is. +/// +/// A string is affected exactly when its two decodings differ: a stored U+FFFD +/// reads the same both times, and a byte that could not be decoded reads U+FFFD +/// in one and [`MARKER`] in the other. Member names are compared the same way. +/// The pointers are built from the lossy tree, so they read as what the operator +/// will see everywhere else. +/// +/// `marked` is `None` -- and every U+FFFD is named, as before -- when the second +/// decode will not parse, and the walk falls back the same way for any subtree +/// whose two sides do not line up. A comparison that could not be made is not +/// grounds for saying less. With [`Marked`] keeping duplicate names, objects +/// always line up; what remains is defence, not a path a document can steer into. +fn replaced_members(lossy: &Value, marked: Option<&Marked>) -> Altered { + let mut found = Altered::default(); + walk(lossy, marked, String::new(), &mut found); + // A member whose name and value were both replaced pushes the same pointer + // twice, and the two pushes are adjacent. + found.all.dedup(); + found.names.dedup(); + found +} + +fn walk(lossy: &Value, marked: Option<&Marked>, pointer: String, found: &mut Altered) { + match lossy { + Value::String(string) => { + let other = match marked { + Some(Marked::String(other)) => Some(other.as_str()), + _ => None, + }; + + if differs(string, other) { + found.all.push(pointer); + } + } + Value::Array(items) => { + let others = match marked { + Some(Marked::Array(others)) if others.len() == items.len() => Some(others), + _ => None, + }; + + for (index, item) in items.iter().enumerate() { + let other = others.map(|others| &others[index]); + walk(item, other, format!("{}/{}", pointer, index), found); + } + } + Value::Object(members) => { + // Both decodes keep the server's member order, and `Unambiguous` has + // already refused any document whose names collide under the lossy + // decode, so the two objects line up by position. By position and + // not by name, because the names are what differ. + let others = match marked { + Some(Marked::Object(others)) if others.len() == members.len() => Some(others), + _ => None, + }; + let mut others = others.map(|others| others.iter()); + + for (name, member) in members { + let other = others.as_mut().and_then(|others| others.next()); + let child = format!("{}/{}", pointer, escape(name)); + + if differs(name, other.map(|(other_name, _)| other_name.as_str())) { + found.all.push(child.clone()); + found.names.push(child.clone()); + } + + walk(member, other.map(|(_, value)| value), child, found); + } + } + _ => {} + } +} + +/// Whether a decoded string holds a byte this decode replaced: it differs from +/// the same string decoded the other way, or -- with nothing to compare against +/// -- it holds a U+FFFD, which is what was reported before the comparison +/// existed. +fn differs(lossy: &str, marked: Option<&str>) -> bool { + match marked { + Some(marked) => lossy != marked, + None => lossy.contains(REPLACEMENT), + } +} + +/// RFC 6901 escaping for a member name used as one pointer segment. +fn escape(name: &str) -> String { + name.replace('~', "~0").replace('/', "~1") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + /// A configuration Unit will happily store and hand back: the value of + /// /routes/0/match/uri holds a raw 0xFF, which no UTF-8 decoder accepts. + fn body_with_raw_byte() -> Vec { + let mut bytes = br#"{"routes":[{"match":{"uri":"/caf"#.to_vec(); + bytes.extend_from_slice(&[0xc3, 0xa9, b'-', 0xff, b'-', b'r', b'a', b'w']); + bytes.extend_from_slice(br#""},"action":{"return":204}}]}"#); + bytes + } + + #[test] + fn valid_utf8_is_decoded_exactly() { + let (value, fidelity) = decode::(r#"{"a":"café"}"#.as_bytes()).expect("valid body must decode"); + assert_eq!(value, serde_json::json!({"a": "café"})); + assert_eq!(fidelity, Fidelity::Exact); + assert!(fidelity.warning("/config").is_none()); + } + + #[test] + fn undecodable_bytes_are_replaced_and_reported() { + let (value, fidelity) = decode::(&body_with_raw_byte()).expect("legacy body must still be readable"); + + // The document is intact apart from the one byte that cannot be + // carried by a Rust string. + assert_eq!(value["routes"][0]["match"]["uri"], "/café-\u{fffd}-raw"); + assert_eq!(value["routes"][0]["action"]["return"], 204); + + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/routes/0/match/uri".to_string()], + names: Vec::new(), + } + ); + + let warning = fidelity.warning("/config").expect("a lossy decode must warn"); + assert!(warning.contains("/config")); + assert!(warning.contains("/routes/0/match/uri")); + assert!(warning.contains("not valid UTF-8")); + } + + /// The typed callers must get the same treatment as `Value` ones: the + /// error-body path deserializes into `Value`, but a caller asking for a + /// map or a `String` goes through `from_value`. + #[test] + fn a_typed_response_is_decoded_lossily_too() { + let mut bytes = br#"{"error":"Unknown parameter \""#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"\"."}"#); + + let (map, fidelity) = decode::>(&bytes).expect("error body must survive"); + assert_eq!(map["error"], "Unknown parameter \"\u{fffd}\"."); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/error".to_string()], + names: Vec::new(), + } + ); + } + + /// Two application names that differ only in their undecodable byte decode + /// to one name, and `serde_json` keeps the last of two equal names: the + /// first application's whole definition would be gone from what is shown, + /// under a warning that mentions only replaced bytes. Fail instead. + #[test] + fn names_that_collide_when_decoded_are_refused() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external","keep":"first"},"app-"#); + bytes.push(0xfe); + bytes.extend_from_slice(br#"":{"type":"python","keep":"second"}}}"#); + + // What the replacing decode on its own produces, and why it cannot be + // shown: one application, and it is not the first one. + let lossy: Value = serde_json::from_str(&String::from_utf8_lossy(&bytes)).expect("the lossy view parses"); + assert_eq!(lossy["applications"].as_object().expect("an object").len(), 1); + + let error = decode::(&bytes).expect_err("a decode that drops a member must fail"); + let message = error.to_string(); + assert!( + message.contains("differ only in bytes that are not valid UTF-8"), + "{}", + message + ); + assert!(message.contains("incomplete"), "{}", message); + } + + /// The collision check must not cost the bodies it is not about: one bad + /// byte in each of two *different* objects collides with nothing. + #[test] + fn replaced_names_in_different_objects_are_not_a_collision() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external"}},"upstreams":{"up-"#); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"servers":{}}}}"#); + + let (value, fidelity) = decode::(&bytes).expect("two separate objects must still decode"); + assert!(value["applications"]["app-\u{fffd}"].is_object()); + assert!(value["upstreams"]["up-\u{fffd}"].is_object()); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec![ + "/applications/app-\u{fffd}".to_string(), + "/upstreams/up-\u{fffd}".to_string() + ], + names: vec![ + "/applications/app-\u{fffd}".to_string(), + "/upstreams/up-\u{fffd}".to_string() + ], + } + ); + } + + /// Two equal *values* are not a collision either: only names are keyed. + #[test] + fn replaced_values_are_never_a_collision() { + let mut bytes = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#""}},{"match":{"uri":"/"#); + bytes.push(0xfe); + bytes.extend_from_slice(br#""}}]}"#); + + let (value, _) = decode::(&bytes).expect("equal values are not equal names"); + assert_eq!(value["routes"].as_array().expect("an array").len(), 2); + } + + /// A body that will not parse as JSON is shown, and a body shown on a + /// terminal is neither unbounded nor trusted. + #[test] + fn an_unparseable_body_is_bounded_and_rendered() { + // A proxy's error page, in the shape that matters: big. + let huge = body_for_display("A".repeat(5 * 1024 * 1024).as_bytes()); + assert!( + huge.len() < MAX_SHOWN_BODY + 128, + "the body must be bounded, got {} bytes", + huge.len() + ); + assert!(huge.starts_with("AAA")); + // The note counts what was measured, which is the output. + assert!(huge.ends_with("[... truncated after 2048 bytes of output]"), "{}", huge); + + // An escape sequence in that page must not reach the terminal, and is + // written out as the escape rather than as U+FFFD, which here would + // claim a byte could not be decoded. Newlines and tabs stay: a page is + // many lines. + let hostile = body_for_display(b"

502\x1b[2J\x1b]0;pwned\x07

\n\tat proxy\r\n"); + assert_eq!( + hostile, + "

502\\u001b[2J\\u001b]0;pwned\\u0007

\n\tat proxy\\u000d\n" + ); + + // Bytes that are not UTF-8 are shown as U+FFFD, which is what U+FFFD + // means here: a byte that could not be decoded. + assert_eq!(body_for_display(&[b'o', b'k', 0xff]), "ok\u{fffd}"); + + // A body that needs neither is passed through unchanged. + assert_eq!(body_for_display(b"502"), "502"); + } + + /// Reading no more of the body than can be shown is the point of the bound, + /// so the rendering must not depend on how much came after it. + #[test] + fn only_as_much_of_a_body_as_can_be_shown_is_read() { + let mut page = b"

502

".to_vec(); + page.resize(8 * 1024 * 1024, b'x'); + + let shown = body_for_display(&page); + assert!(shown.starts_with("

502

xxx")); + assert!(shown.len() < MAX_SHOWN_BODY + 128); + + // A character the cut lands inside must not break the rendering. + let mut multibyte = "caf\u{e9}".repeat(2000).into_bytes(); + multibyte.truncate(MAX_SHOWN_BODY + 1); + let shown = body_for_display(&multibyte); + assert!(shown.starts_with("caf\u{e9}caf\u{e9}"), "{}", &shown[..16]); + assert!(shown.len() < MAX_SHOWN_BODY + 128); + } + + /// A member name can hold any character JSON can spell, including the + /// escape for ESC, and every warning interpolates the pointer into stderr. + /// An application named with an ANSI sequence must not be able to clear the + /// operator's screen, and one named with a newline must not be able to forge + /// a line that looks like unitctl's own. + /// + /// The rendering is asserted, not merely the absence of control characters: + /// an absence passes for U+FFFD too, and U+FFFD here would claim the name + /// holds a byte the server could not hand over, which it does not. + #[test] + fn a_pointer_cannot_carry_control_characters_to_the_terminal() { + // The name is the six characters backslash-u-0-0-1-b and so on: what a + // server would send, not a raw control byte, which the parser refuses. + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"{\"app"); + bytes.extend_from_slice(b"\\u001b[2J"); + bytes.extend_from_slice(b"\\u0007"); + bytes.extend_from_slice(b"\\nWarning: all is well"); + bytes.extend_from_slice(b"\\r\\t\":\"caf"); + bytes.push(0xff); + bytes.extend_from_slice(b"\"}"); + + // serde_json really does decode those into control characters: without + // that this test would be proving nothing. + let decoded: Value = serde_json::from_str(&String::from_utf8_lossy(&bytes)).expect("the body parses"); + let name = decoded.as_object().expect("an object").keys().next().expect("a name"); + assert!(name.contains('\u{1b}'), "serde must decode the escape: {:?}", name); + assert!(name.contains('\n')); + + let (_, fidelity) = decode::(&bytes).expect("the body must still be readable"); + + // Every control character is written out, as the escape and not as + // U+FFFD, and the ordinary text around them still identifies the member. + let listed = fidelity.members(); + assert_eq!(listed, "/app\\u001b[2J\\u0007\\u000aWarning: all is well\\u000d\\u0009"); + assert!( + !listed.contains(REPLACEMENT), + "a name that is valid UTF-8 must not be shown as a replaced byte: {:?}", + listed + ); + + let warning = fidelity.warning("/config").expect("a lossy decode must warn"); + assert!(warning.ends_with(&listed), "{}", warning); + // The warning's own two lines are the only newlines in it. + assert_eq!(warning.matches('\n').count(), 1, "a newline survived: {:?}", warning); + } + + /// The percent sign was once how this check told undecodable bytes apart, + /// and JSON can spell a percent as an escape, so neither a literal `%FF` in + /// a name nor an escaped one may hide a collision. Asking the question of + /// the decoded names is what makes the spelling irrelevant. + #[test] + fn no_spelling_of_a_name_hides_a_collision() { + let mut literal = br#"{"applications":{"app-"#.to_vec(); + literal.push(0xff); + literal.extend_from_slice(br#"":{"keep":"raw-ff"},"app-"#); + literal.push(0xfe); + literal.extend_from_slice(br#"":{"keep":"raw-fe"},"app-%FF":{"keep":"literal"}}}"#); + + let mut escaped = br#"{"applications":{"app-"#.to_vec(); + escaped.push(0xff); + escaped.extend_from_slice(br#"":{"keep":"raw-ff"},"app-"#); + escaped.push(0xfe); + escaped.extend_from_slice(br#"":{"keep":"raw-fe"},"app-"#); + // An escaped percent: the six characters backslash-u-0-0-2-5. + escaped.extend_from_slice(b"\\u0025"); + escaped.extend_from_slice(br#"FF":{"keep":"escaped"}}}"#); + + for bytes in [&literal, &escaped] { + // Three names went in and the replacing decode keeps two, so the + // body really is one a decode would silently truncate. + let lossy: Value = serde_json::from_str(&String::from_utf8_lossy(bytes)).expect("the lossy view parses"); + assert_eq!(lossy["applications"].as_object().expect("an object").len(), 2); + + let error = decode::(bytes).expect_err("a decode that drops a member must fail"); + assert!( + error + .to_string() + .contains("differ only in bytes that are not valid UTF-8"), + "{}", + error + ); + } + } + + /// The one collision where neither name was damaged by the decode: an + /// operator can legitimately store a name holding a real U+FFFD (`EF BF BD`), + /// and a sibling holding a raw undecodable byte decodes to the same thing. + #[test] + fn a_stored_replacement_character_collides_with_a_replaced_byte() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.extend_from_slice("\u{fffd}".as_bytes()); + bytes.extend_from_slice(br#"":{"keep":"stored"},"app-"#); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"keep":"raw"}}}"#); + + // The stored name is valid UTF-8 and arrives untouched; it is the raw + // byte that decodes onto it. + let lossy: Value = serde_json::from_str(&String::from_utf8_lossy(&bytes)).expect("the lossy view parses"); + assert_eq!(lossy["applications"].as_object().expect("an object").len(), 1); + + let error = decode::(&bytes).expect_err("one of the two would be dropped"); + assert!(error.to_string().contains("share a name"), "{}", error); + } + + /// Duplicate names are refused whatever made them duplicate, so the refusal + /// must not blame undecodable bytes as though it had checked. + /// + /// A body that is valid UTF-8 never reaches the wrapper at all: the strict + /// decode succeeds first, dropping a member as `serde_json` always has. + /// Only a body that also holds an undecodable byte somewhere gets here. + #[test] + fn the_refusal_does_not_blame_a_cause_it_did_not_check() { + let clean = br#"{"applications":{"a":1,"a":2},"x":"ok"}"#; + let (value, fidelity) = decode::(clean).expect("the strict decode accepts duplicates"); + assert!(fidelity.is_exact(), "a valid-UTF-8 body must not reach the wrapper"); + assert_eq!(value["applications"]["a"], 2); + + let mut dirty = br#"{"applications":{"a":1,"a":2},"x":"caf"#.to_vec(); + dirty.push(0xff); + dirty.extend_from_slice(br#""}"#); + + let message = decode::(&dirty) + .expect_err("a duplicate name is still a dropped member") + .to_string(); + assert!( + message.contains("two members of one object share a name"), + "{}", + message + ); + // The mechanism is offered as the usual reason, not asserted as the cause. + assert!(message.contains("the usual reason is"), "{}", message); + assert!( + message.contains("differ only in bytes that are not valid UTF-8"), + "{}", + message + ); + assert!(message.contains("unitctl export"), "{}", message); + } + + /// The refusal is about names, at every depth, and nothing else: a duplicate + /// inside a nested object or inside an array element is the same loss. + #[test] + fn a_collision_is_caught_at_every_depth() { + let mut nested = br#"{"routes":{"a":{"b":{"app-"#.to_vec(); + nested.push(0xff); + nested.extend_from_slice(br#"":1,"app-"#); + nested.push(0xfe); + nested.extend_from_slice(br#"":2}}}}"#); + assert!(decode::(&nested).is_err(), "a nested collision must be caught"); + + let mut in_array = br#"{"routes":[{"app-"#.to_vec(); + in_array.push(0xff); + in_array.extend_from_slice(br#"":1,"app-"#); + in_array.push(0xfe); + in_array.extend_from_slice(br#"":2}]}"#); + assert!( + decode::(&in_array).is_err(), + "a collision inside an array element must be caught" + ); + } + + /// A sequence truncated mid-character is one undecodable run, not a reason + /// to refuse: there is one name, so there is nothing ambiguous about it. + #[test] + fn bytes_that_end_mid_character_still_decode() { + let mut bytes = br#"{"routes":[{"match":{"uri":"/caf"#.to_vec(); + bytes.extend_from_slice(&[0xe2, 0x82]); + bytes.extend_from_slice(br#""}}]}"#); + + let (value, fidelity) = decode::(&bytes).expect("one truncated character is not a collision"); + assert_eq!(value["routes"][0]["match"]["uri"], "/caf\u{fffd}"); + assert!(!fidelity.is_exact()); + } + + /// `decode` is generic, and the callers that read a configuration ask for a + /// map, which is what refuses a 200 carrying `null`, an array or a bare + /// string. That has to hold on the lossy path too, since a body with an + /// undecodable byte reaches `T` through `from_value` instead of directly. + #[test] + fn a_configuration_shape_is_required_on_both_paths() { + type Config = std::collections::BTreeMap; + + let (config, fidelity) = + decode::(br#"{"listeners":{"*:8080":{"pass":"routes"}}}"#).expect("an object is a configuration"); + assert_eq!(config.keys().collect::>(), vec!["listeners"]); + assert!(fidelity.is_exact()); + + for body in [&b"null"[..], b"[]", b"[{\"listeners\":{}}]", b"\"a string\"", b"204"] { + assert!( + decode::(body).is_err(), + "{} is not a configuration", + String::from_utf8_lossy(body) + ); + // And why the type the caller asks for is what does the refusing: + // this function cannot, because `Value` accepts every shape. + assert!(decode::(body).is_ok()); + } + + // The same bodies with an undecodable byte in them take the lossy path, + // and must be refused there as well. + let mut array = br#"["caf"#.to_vec(); + array.push(0xff); + array.extend_from_slice(br#""]"#); + assert!(std::str::from_utf8(&array).is_err()); + assert!(decode::(&array).is_err(), "an array is not a configuration"); + + let mut string = br#""caf"#.to_vec(); + string.push(0xff); + string.push(b'"'); + assert!( + decode::(&string).is_err(), + "a bare string is not a configuration" + ); + + // And an object with such a byte still decodes, lossily. + let mut object = br#"{"caf"#.to_vec(); + object.push(0xff); + object.extend_from_slice(br#"":{}}"#); + let (_, fidelity) = decode::(&object).expect("an object is still a configuration"); + assert!(!fidelity.is_exact()); + } + + /// `EF BF BD` is valid UTF-8, so a U+FFFD an operator stored arrives intact + /// and was never replaced. Naming it sends them to `edit`'s recovery advice + /// for a member that is not damaged, and the point of the warning is to say + /// which member is. + #[test] + fn a_stored_replacement_character_is_not_reported_as_replaced() { + let mut bytes = br#"{"valid":"stored: "#.to_vec(); + bytes.extend_from_slice(&[0xef, 0xbf, 0xbd]); + bytes.extend_from_slice(br#"","legacy":"raw: "#); + bytes.push(0xff); + bytes.extend_from_slice(br#""}"#); + + let (value, fidelity) = decode::(&bytes).expect("the body must still be readable"); + + // Both values read as U+FFFD, which is exactly why they cannot be told + // apart by looking at the decoded text. + assert_eq!(value["valid"], "stored: \u{fffd}"); + assert_eq!(value["legacy"], "raw: \u{fffd}"); + + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/legacy".to_string()], + names: Vec::new(), + } + ); + + let warning = fidelity.warning("/config").expect("a lossy decode must warn"); + assert!(warning.contains("/legacy"), "{}", warning); + assert!( + !warning.contains("/valid"), + "an intact member must not be named: {}", + warning + ); + } + + /// The same distinction for a member *name*: one name holds a stored U+FFFD + /// and its sibling holds a byte that could not be decoded. They are + /// different names, so nothing collides, and only one was altered. + #[test] + fn a_stored_replacement_character_in_a_name_is_not_reported() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.extend_from_slice(&[0xef, 0xbf, 0xbd]); + bytes.extend_from_slice(br#"-stored":{"type":"external"},"other-"#); + bytes.push(0xff); + bytes.extend_from_slice(br#"-raw":{"type":"python"}}}"#); + + let (_, fidelity) = decode::(&bytes).expect("two different names do not collide"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/applications/other-\u{fffd}-raw".to_string()], + names: vec!["/applications/other-\u{fffd}-raw".to_string()], + } + ); + } + + /// With nothing to compare against, every U+FFFD is named, which is what was + /// reported before the second decode existed. A comparison that could not be + /// made is not grounds for saying less. + #[test] + fn without_a_second_decode_every_replacement_is_named() { + let lossy = serde_json::json!({ + "valid": "stored: \u{fffd}", + "legacy": "raw: \u{fffd}", + "clean": "nothing here", + }); + + let altered = replaced_members(&lossy, None); + assert_eq!(altered.all, vec!["/valid".to_string(), "/legacy".to_string()]); + assert!(altered.names.is_empty(), "no member *name* holds one here"); + } + + /// Nothing searches the second decode for its marker, so content that is + /// the marker -- or JSON's escaping spelling it out, which no check of the + /// raw bytes would see -- cannot make an intact member look replaced. + #[test] + fn content_that_looks_like_the_marker_is_not_reported() { + let mut literal = br#"{"innocent":"?","legacy":"raw: "#.to_vec(); + literal.push(0xff); + literal.extend_from_slice(br#""}"#); + + let mut escaped = br#"{"innocent":""#.to_vec(); + escaped.extend_from_slice(b"\\u003f"); + escaped.extend_from_slice(br#"","legacy":"raw: "#); + escaped.push(0xff); + escaped.extend_from_slice(br#""}"#); + + for bytes in [&literal, &escaped] { + let (value, fidelity) = decode::(bytes).expect("the body must decode"); + assert_eq!(value["innocent"], "?", "the marker text decodes to itself"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/legacy".to_string()], + names: Vec::new(), + }, + "only the member with an undecodable byte is named" + ); + } + } + + /// The marker is ordinary text, so a document can hold a member whose name + /// *is* it beside one whose name decodes to it. Keeping duplicate names on + /// the marked side is what stops that from collapsing the two sides into + /// different lengths and falling back to naming every U+FFFD in the object. + #[test] + fn a_name_that_is_the_marker_does_not_disturb_the_comparison() { + let mut bytes = br#"{"applications":{"?":{"note":"literal question mark"},""#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"note":"raw byte"},"valid":{"note":"stored: "#); + bytes.extend_from_slice(&[0xef, 0xbf, 0xbd]); + bytes.extend_from_slice(br#""}}}"#); + + // Three names go in, and two of them read as "?" once the bytes that + // could not be decoded are written as the marker. + let lossy: Value = serde_json::from_str(&String::from_utf8_lossy(&bytes)).expect("the lossy view parses"); + assert_eq!(lossy["applications"].as_object().expect("an object").len(), 3); + + let (_, fidelity) = decode::(&bytes).expect("the body must decode"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/applications/\u{fffd}".to_string()], + names: vec!["/applications/\u{fffd}".to_string()], + }, + "only the name holding a byte that could not be decoded was altered" + ); + + // The two members that were not touched: the one literally named "?", + // and the one whose value holds a U+FFFD the operator stored. + let warning = fidelity.warning("/config").expect("a lossy decode must warn"); + assert!(!warning.contains("/valid"), "an intact member was named: {}", warning); + } + + /// A pointer whose *name* was altered cannot be PUT to: it shows U+FFFD + /// where the server holds a byte, so it names a member that does not exist. + /// The two kinds are kept apart so the advice can say so. + #[test] + fn a_replaced_name_is_reported_apart_from_a_replaced_value() { + let mut in_a_name = br#"{"applications":{"app-"#.to_vec(); + in_a_name.push(0xff); + in_a_name.extend_from_slice(br#"":{"type":"external"}}}"#); + + let (_, fidelity) = decode::(&in_a_name).expect("the body must decode"); + assert_eq!(fidelity.members(), "/applications/app-\u{fffd}"); + assert_eq!( + fidelity.replaced_names(), + "/applications/app-\u{fffd}", + "a replaced name must be reported as one" + ); + + let mut in_a_value = br#"{"routes":[{"match":{"uri":"/caf"#.to_vec(); + in_a_value.push(0xff); + in_a_value.extend_from_slice(br#""}}]}"#); + + let (_, fidelity) = decode::(&in_a_value).expect("the body must decode"); + assert_eq!(fidelity.members(), "/routes/0/match/uri"); + assert!( + fidelity.replaced_names().is_empty(), + "a replaced value is repairable by a PUT and must not be listed as a name" + ); + + // An exact body has neither. + let (_, fidelity) = decode::(br#"{"routes":[]}"#).expect("valid UTF-8 decodes exactly"); + assert!(fidelity.replaced_names().is_empty()); + } + + /// The two lists are what the advice branches on, so each must hold only its + /// own kind. + #[test] + fn values_and_names_are_listed_apart() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external"}},"routes":[{"uri":"/caf"#); + bytes.push(0xfe); + bytes.extend_from_slice(br#""}]}"#); + + let (_, fidelity) = decode::(&bytes).expect("the body decodes"); + assert_eq!(fidelity.replaced_values(), "/routes/0/uri"); + assert_eq!(fidelity.replaced_names(), "/applications/app-\u{fffd}"); + assert_eq!( + fidelity.members(), + "/applications/app-\u{fffd}, /routes/0/uri", + "the headline still lists both" + ); + + // A body with only a replaced value has no names, and the reverse. + let mut value_only = br#"{"routes":[{"uri":"/caf"#.to_vec(); + value_only.push(0xff); + value_only.extend_from_slice(br#""}]}"#); + let (_, fidelity) = decode::(&value_only).expect("the body decodes"); + assert_eq!(fidelity.replaced_values(), "/routes/0/uri"); + assert!(fidelity.replaced_names().is_empty()); + + // And an exact body has neither. + let (_, fidelity) = decode::(br#"{"routes":[]}"#).expect("valid UTF-8 decodes exactly"); + assert!(fidelity.replaced_values().is_empty()); + assert!(fidelity.replaced_names().is_empty()); + } + + /// Both at once: one member's name was altered and another's value was, and + /// each is reported in its own list. + #[test] + fn a_name_and_a_value_are_reported_separately() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external"}},"routes":[{"uri":"/caf"#); + bytes.push(0xfe); + bytes.extend_from_slice(br#""}]}"#); + + let (_, fidelity) = decode::(&bytes).expect("the body must decode"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/applications/app-\u{fffd}".to_string(), "/routes/0/uri".to_string()], + names: vec!["/applications/app-\u{fffd}".to_string()], + } + ); + } + + /// serde_json must keep the order the server sent. The comparison in + /// `replaced_members` pairs the two decodes member by member, in order, so a + /// sorted map would pair them wrongly. The names here are not in sorted + /// order, so this fails if the preserve_order feature is ever turned off. + #[test] + fn member_order_survives_a_round_trip() { + let source = r#"{"z":1,"a":2,"m":3}"#; + let parsed: Value = serde_json::from_str(source).expect("the body parses"); + + let names: Vec<&str> = parsed + .as_object() + .expect("an object") + .keys() + .map(String::as_str) + .collect(); + assert_eq!(names, vec!["z", "a", "m"], "serde_json must keep member order"); + + assert_eq!(serde_json::to_string(&parsed).expect("it serializes"), source); + } + + /// The same precondition, at the place that depends on it: the two decodes + /// are paired by position. With a sorted map the damaged value and the + /// undamaged one swap places, and both are reported. + #[test] + fn members_out_of_sorted_order_are_paired_correctly() { + let mut bytes = br#"{"z":"ok","a":"raw: "#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#""}"#); + + let (_, fidelity) = decode::(&bytes).expect("the body decodes"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/a".to_string()], + names: Vec::new(), + }, + "only /a holds a byte that could not be decoded, and it is a value" + ); + } + + /// A member *name* can hold the raw bytes just as a value can, and an + /// application name is exactly such a member. + #[test] + fn a_replaced_member_name_is_pointed_at_once() { + let mut bytes = br#"{"applications":{"app-"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#"":{"type":"external"}}}"#); + + let (_, fidelity) = decode::(&bytes).expect("body must survive"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/applications/app-\u{fffd}".to_string()], + names: vec!["/applications/app-\u{fffd}".to_string()], + } + ); + } + + /// `/` and `~` in a member name are pointer syntax and must be escaped, or + /// the pointer names a member that does not exist. + #[test] + fn pointer_segments_are_escaped() { + let mut bytes = br#"{"listeners":{"*:8080":{"pass":"routes/a~b/"#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#""}}}"#); + + let (_, fidelity) = decode::(&bytes).expect("body must survive"); + assert_eq!( + fidelity, + Fidelity::Replaced { + members: vec!["/listeners/*:8080/pass".to_string()], + names: Vec::new(), + } + ); + + assert_eq!(escape("a/b~c"), "a~1b~0c"); + } + + /// Tolerating undecodable bytes must not turn malformed JSON into a + /// success: a body that is valid UTF-8 fails exactly as it did before. + #[test] + fn malformed_utf8_json_still_fails() { + assert!(decode::(br#"{"a":"#).is_err()); + assert!(decode::(b"not json at all").is_err()); + // Undecodable *and* malformed is still an error. + assert!(decode::(&[b'{', 0xff]).is_err()); + } + + /// A body that decodes but does not match the type asked for keeps failing + /// too, rather than being silently coerced. + #[test] + fn a_type_mismatch_is_not_hidden() { + assert!(decode::>(br#"{"a":1}"#).is_err()); + let mut bytes = br#"{"a":1,"b":""#.to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(br#""}"#); + assert!(decode::>(&bytes).is_err()); + } + + /// The defensive arm: a lossy decode that located nothing still has to say + /// something, rather than printing an empty list. + #[test] + fn a_lossy_decode_that_located_nothing_still_says_so() { + let listed = Fidelity::Replaced { + members: Vec::new(), + names: Vec::new(), + } + .members(); + assert_eq!(listed, "an unlocatable part of the document"); + assert!(Fidelity::Replaced { + members: Vec::new(), + names: Vec::new() + } + .warning("/config") + .expect("a lossy decode must warn") + .contains("an unlocatable part of the document")); + } + + #[test] + fn a_long_list_of_members_is_capped() { + let members: Vec = (0..12).map(|i| format!("/m{}", i)).collect(); + let listed = Fidelity::Replaced { + members, + names: Vec::new(), + } + .members(); + assert!(listed.starts_with("/m0, /m1,")); + assert!(listed.ends_with("and 4 more")); + } +} diff --git a/tools/unitctl/unit-client-rs/src/lib.rs b/tools/unitctl/unit-client-rs/src/lib.rs index 59dd26813..6022463c7 100644 --- a/tools/unitctl/unit-client-rs/src/lib.rs +++ b/tools/unitctl/unit-client-rs/src/lib.rs @@ -5,6 +5,7 @@ extern crate hyperlocal; extern crate serde; extern crate serde_json; pub mod control_socket_address; +pub mod json_body; mod runtime_flags; pub mod unit_client; mod unitd_cmd; diff --git a/tools/unitctl/unit-client-rs/src/unit_client.rs b/tools/unitctl/unit-client-rs/src/unit_client.rs index e469546d1..8a908107c 100644 --- a/tools/unitctl/unit-client-rs/src/unit_client.rs +++ b/tools/unitctl/unit-client-rs/src/unit_client.rs @@ -4,7 +4,7 @@ use std::fmt::Debug; use std::rc::Rc; use std::{fmt, io}; -use bytes::{Buf, Bytes}; +use bytes::Bytes; use custom_error::custom_error; use http_body_util::{BodyExt, Full}; use hyper::{http, Request}; @@ -16,6 +16,7 @@ use hyperlocal::UnixConnector; use serde::{Deserialize, Serialize}; use crate::control_socket_address::ControlSocket; +use crate::json_body::{self, Fidelity}; use unit_openapi::apis::configuration::Configuration; use unit_openapi::apis::{ ApplicationsApi, ApplicationsApiClient, AppsApi, AppsApiClient, Error as OpenAPIError, ListenersApi, @@ -28,7 +29,10 @@ const USER_AGENT: &str = concat!("Unit CLI/", env!("CARGO_PKG_VERSION"), "/rust" custom_error! {pub UnitClientError OpenAPIError { source: OpenAPIError } = "OpenAPI error", JsonError { source: serde_json::Error, - path: String} = "JSON error [path={path}]", + path: String} = "JSON error [path={path}]: {source}", + UnparseableBodyError { source: serde_json::Error, + path: String, + body: String} = "JSON error [path={path}]: {source}\n{body}", HyperError { source: hyper_util::client::legacy::Error, control_socket_address: String, path: String} = "Communications error [control_socket_address={control_socket_address}, path={path}]: {source}", @@ -167,17 +171,27 @@ pub struct UnitClient { client: Box, } +/// Say out loud when a body had to be decoded with bytes replaced, so that +/// nobody mistakes what is printed for what the server stores. +fn warn_if_replaced(fidelity: &Fidelity, path: &str) { + if let Some(warning) = fidelity.warning(path) { + eprintln!("{}", warning); + } +} + /// Pulls the parts of a control API error body out for display: the message, the /// detail, where the error is, and the name a typo probably meant. The last two /// come back pre-formatted, empty when they do not apply, so a response carrying /// neither renders exactly as it did before they existed. fn describe_error_body(body: &serde_json::Value) -> (String, String, String, String) { - let member = |name: &str| { - body.get(name) - .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .to_string() - }; + // Whatever answered wrote these strings, and they are printed to a + // terminal. A genuine unitd echoes only the requester's own member name + // here (src/nxt_conf_validation.c), so the writer to guard against is the + // same intermediary the unparseable-body path already guards against: an + // escape sequence in a 400 would otherwise reach the screen, and a newline + // would let it forge a line of unitctl's own output. + let member = + |name: &str| json_body::one_line(body.get(name).and_then(serde_json::Value::as_str).unwrap_or_default()); let mut error = member("error"); if error.is_empty() { @@ -192,7 +206,7 @@ fn describe_error_body(body: &serde_json::Value) -> (String, String, String, Str match location.get("path").and_then(serde_json::Value::as_str) { Some("") => Some("the document root".to_string()), - Some(pointer) => Some(pointer.to_string()), + Some(pointer) => Some(json_body::one_line(pointer)), None => match (number("line"), number("column"), number("offset")) { (Some(line), Some(column), _) => Some(format!("line {}, column {}", line, column)), (_, _, Some(offset)) => Some(format!("byte offset {}", offset)), @@ -201,7 +215,9 @@ fn describe_error_body(body: &serde_json::Value) -> (String, String, String, Str } }); - let location = place.map(|place| format!("\n Location: {}", place)).unwrap_or_default(); + let location = place + .map(|place| format!("\n Location: {}", place)) + .unwrap_or_default(); let suggestion = match member("suggestion").as_str() { "" => String::new(), @@ -244,11 +260,17 @@ impl UnitClient { } } - /// Sends a request to Unit and deserializes the JSON response body into the value of type `RESPONSE`. - pub async fn send_request_and_deserialize_response serde::Deserialize<'de>>( + /// Sends a request to Unit and returns the response body as it arrived. + /// + /// A caller that will write the document back needs the bytes the server + /// sent and not a re-encoding of them, because a configuration can hold + /// bytes no Rust `String` can carry. Turning a non-success status into an + /// error happens here so that those callers share it with the + /// deserializing variant below. + pub async fn send_request_and_collect_body( &self, mut request: Request>, - ) -> Result { + ) -> Result { let uri = request.uri().clone(); let path: &str = uri.path(); @@ -272,7 +294,6 @@ impl UnitClient { })? .to_bytes(); - let mut reader = body_bytes.reader(); if !status.is_success() { // The control API answers an error with "error" and, where they // apply, "detail", a "location" object placing the error, and a @@ -280,11 +301,22 @@ impl UnitClient { // typo of. "location" is an object, so the body is not a map of // strings and must not be deserialized as one: that fails the // whole body and hides the message it carries. - let body: serde_json::Value = - serde_json::from_reader(&mut reader).map_err(|error| UnitClientError::JsonError { - source: error, - path: path.to_string(), - })?; + let (body, fidelity) = match json_body::decode::(&body_bytes) { + Ok(decoded) => decoded, + // Not JSON at all -- an intermediary's page, or a truncated + // answer. A body that will not parse is exactly when its text + // matters most, so show it instead of a parser complaint, but + // show it bounded and with the control characters taken out. + Err(_) => { + return Err(UnitClientError::HttpResponseError { + status, + path: path.to_string(), + body: json_body::body_for_display(&body_bytes), + }) + } + }; + + warn_if_replaced(&fidelity, path); let (error, detail, location, suggestion) = describe_error_body(&body); @@ -297,10 +329,32 @@ impl UnitClient { suggestion, }); } - serde_json::from_reader(&mut reader).map_err(|error| UnitClientError::JsonError { - source: error, - path: path.to_string(), - }) + + Ok(body_bytes) + } + + /// Sends a request to Unit and deserializes the JSON response body into the value of type `RESPONSE`. + pub async fn send_request_and_deserialize_response serde::Deserialize<'de>>( + &self, + request: Request>, + ) -> Result { + let path = request.uri().path().to_string(); + let body_bytes = self.send_request_and_collect_body(request).await?; + + // A success status is no guarantee that Unit answered: a proxy or a + // captive portal can return 200 with a page of its own. Show it, for + // the reason the non-success arm above shows its body -- a body that + // will not parse is exactly when its text matters most. + let (response, fidelity) = + json_body::decode(&body_bytes).map_err(|error| UnitClientError::UnparseableBodyError { + source: error, + path: path.clone(), + body: json_body::body_for_display(&body_bytes), + })?; + + warn_if_replaced(&fidelity, &path); + + Ok(response) } pub fn listeners_api(&self) -> Box { @@ -384,7 +438,14 @@ impl UnitClient { } } -pub type UnitSerializableMap = HashMap; +/// A JSON object as it travels between unitctl and the control API. +/// +/// `serde_json::Map` and not `HashMap`: with the crate's `preserve_order` +/// feature this keeps the server's member order, and a `HashMap` does not. A +/// `HashMap` here reordered the members of `/config` on every single run, so +/// two `unitctl export`s of one unchanged configuration disagreed, and +/// `unitctl edit` presented a document shuffled differently each time. +pub type UnitSerializableMap = serde_json::Map; #[derive(Clone, Debug, Deserialize, Serialize)] pub struct UnitStatus { @@ -427,6 +488,43 @@ mod tests { use super::*; + /// The four strings a control API error body carries are the server's text, + /// and they print straight to stderr. The rendering is asserted rather than + /// the absence of control characters: an absence is also satisfied by U+FFFD, + /// which would say these strings hold bytes that could not be decoded. + #[test] + fn an_error_body_cannot_carry_control_characters_to_the_terminal() { + let esc = String::from_utf8(vec![0x1b]).unwrap(); + let bell = String::from_utf8(vec![0x07]).unwrap(); + + let (error, detail, location, suggestion) = describe_error_body(&serde_json::json!({ + "error": format!("{}[2Jcleared", esc), + "detail": format!("{}pwned\nWarning: forged line", bell), + "location": {"path": format!("/routes/{}[31m", esc)}, + "suggestion": format!("pass{}", esc), + })); + + assert_eq!(error, "\\u001b[2Jcleared"); + assert_eq!(detail, "\\u0007pwned\\u000aWarning: forged line"); + assert_eq!(location, "\n Location: /routes/\\u001b[31m"); + assert_eq!(suggestion, "\n Did you mean: pass\\u001b"); + + for part in [&error, &detail, &location, &suggestion] { + assert!( + !part.contains('\u{fffd}'), + "text that is valid UTF-8 must not be shown as a replaced byte: {:?}", + part + ); + } + + // Location and suggestion are built with a deliberate leading newline, + // and it is the only one either may have. + assert_eq!(location.matches('\n').count(), 1); + assert_eq!(suggestion.matches('\n').count(), 1); + assert!(!error.contains('\n')); + assert!(!detail.contains('\n'), "detail forged a line: {:?}", detail); + } + /// The shapes nxt_controller_response() can send, and how each renders. #[test] fn describes_every_error_body_shape() { @@ -483,6 +581,64 @@ mod tests { assert_eq!(case(serde_json::json!("not an object")), "Unknown error|||"); } + /// A 200 is no guarantee that Unit answered: a proxy or a captive portal can + /// return one with a page of its own. The parser's complaint about that page + /// says nothing about it, so the page is shown as well -- bounded and with + /// its control characters written out, since it is a stranger's text. + #[test] + fn an_unparseable_success_body_is_shown_not_just_complained_about() { + let page = b"captive portal: sign in at http://portal.example/\x1b[2J"; + let source = serde_json::from_slice::(page).expect_err("not JSON"); + + let error = UnitClientError::UnparseableBodyError { + path: "/config".to_string(), + source, + body: json_body::body_for_display(page), + }; + let rendered = error.to_string(); + + assert!(rendered.contains("/config"), "{}", rendered); + assert!( + rendered.contains("captive portal: sign in"), + "the page must be shown: {}", + rendered + ); + assert!( + rendered.contains("expected value"), + "the parser's reason stays: {}", + rendered + ); + assert!( + !rendered.contains('\u{1b}'), + "an escape reached the message: {:?}", + rendered + ); + assert!( + rendered.contains("\\u001b[2J"), + "the escape is written out: {}", + rendered + ); + } + + /// The error that carries such a body must carry it as shown: the body is + /// embedded in the message the user reads. + #[test] + fn the_http_response_error_shows_the_body_it_was_given() { + let error = UnitClientError::HttpResponseError { + status: http::StatusCode::BAD_GATEWAY, + path: "/config".to_string(), + body: json_body::body_for_display(&[0x1b, b'[', b'2', b'J']), + }; + let rendered = error.to_string(); + assert!(rendered.contains("/config")); + assert!(rendered.contains("502")); + assert!( + !rendered.contains('\u{1b}'), + "an escape reached the message: {}", + rendered + ); + } + // Integration tests #[tokio::test] diff --git a/tools/unitctl/unitctl/src/cmd/edit.rs b/tools/unitctl/unitctl/src/cmd/edit.rs index 64f2806d7..60c3f9493 100644 --- a/tools/unitctl/unitctl/src/cmd/edit.rs +++ b/tools/unitctl/unitctl/src/cmd/edit.rs @@ -1,10 +1,11 @@ use crate::inputfile::{InputFile, InputFormat}; -use crate::requests::{send_and_validate_config_deserialize_response, send_empty_body_deserialize_response}; +use crate::requests::{send_and_validate_config_deserialize_response, send_empty_body_read_bytes}; use crate::unitctl::UnitCtl; use crate::unitctl_error::ControlSocketErrorKind; use crate::{wait, OutputFormat, UnitctlError}; use std::path::{Path, PathBuf}; -use unit_client_rs::unit_client::UnitClient; +use unit_client_rs::json_body; +use unit_client_rs::unit_client::{UnitClient, UnitSerializableMap}; use which::which; const EDITOR_ENV_VARS: [&str; 2] = ["EDITOR", "VISUAL"]; @@ -20,7 +21,11 @@ const EDITOR_KNOWN_LIST: [&str; 8] = [ ]; pub(crate) async fn cmd(cli: &UnitCtl, output_format: OutputFormat) -> Result<(), UnitctlError> { - if cli.control_socket_addresses.as_ref().map_or(false, |addrs| addrs.len() > 1) { + if cli + .control_socket_addresses + .as_ref() + .map_or(false, |addrs| addrs.len() > 1) + { return Err(UnitctlError::ControlSocketError { kind: ControlSocketErrorKind::General, message: "too many control sockets. specify at most one.".to_string(), @@ -29,8 +34,36 @@ pub(crate) async fn cmd(cli: &UnitCtl, output_format: OutputFormat) -> Result<() let mut control_sockets = wait::wait_for_sockets(cli).await?; let client = UnitClient::new(control_sockets.pop().unwrap()); - // Get latest configuration - let current_config = send_empty_body_deserialize_response(&client, "GET", "/config").await?; + // Get latest configuration. Editing means handing it back, and the way + // back runs the file through serde_json, whose strings are Rust strings: + // bytes Unit stores but UTF-8 cannot carry would be replaced by the mere + // act of opening the editor. Refuse rather than rewrite a configuration + // the user only meant to look at. + // UnitSerializableMap and not serde_json::Value: a configuration is a JSON + // object, and the type is what keeps a 200 carrying "null", an array or a + // bare string -- an intermediary talking -- from being opened in the editor + // and handed back as the whole configuration. + let raw_config = send_empty_body_read_bytes(&client, "GET", "/config").await?; + let current_config: UnitSerializableMap = match json_body::decode(&raw_config) { + Ok((config, fidelity)) if fidelity.is_exact() => config, + Ok((_, fidelity)) => { + return Err(UnitctlError::UndecodableConfiguration { + path: "/config".to_string(), + members: fidelity.members(), + values: fidelity.replaced_values(), + names: fidelity.replaced_names(), + }) + } + // Name the path, as the JsonError this replaced did: "expected value at + // line 1 column 1" on its own does not say what was being read -- and + // show the body, because a 200 carrying an intermediary's page is the + // ordinary way this arm is reached. + Err(error) => { + return Err(UnitctlError::DeserializationError { + message: format!("/config: {}\n{}", error, json_body::body_for_display(&raw_config)), + }) + } + }; // Write JSON to temporary file - this file will automatically be deleted by the OS when // the last file handle to it is removed. diff --git a/tools/unitctl/unitctl/src/cmd/save.rs b/tools/unitctl/unitctl/src/cmd/save.rs index 075f0cbe2..b0efb9fb3 100644 --- a/tools/unitctl/unitctl/src/cmd/save.rs +++ b/tools/unitctl/unitctl/src/cmd/save.rs @@ -1,4 +1,4 @@ -use crate::requests::send_empty_body_deserialize_response; +use crate::requests::send_empty_body_read_bytes; use crate::unitctl::UnitCtl; use crate::unitctl_error::ControlSocketErrorKind; use crate::wait; @@ -6,10 +6,62 @@ use crate::UnitctlError; use std::fs::File; use std::io::stdout; use tar::{Builder, Header}; -use unit_client_rs::unit_client::UnitClient; +use unit_client_rs::json_body; +use unit_client_rs::unit_client::{UnitClient, UnitSerializableMap}; + +/// Pretty-print the saved document, and keep the server's own bytes when a Rust +/// string cannot carry them. +/// +/// A configuration can hold bytes that are not valid UTF-8, and re-encoding +/// those through `serde_json` would replace them. A backup that differs from +/// the configuration it backs up is worse than an unformatted one. +/// +/// Whichever way the body is read, it has to be a configuration to be stored as +/// one, and `UnitSerializableMap` is what says so: a configuration is a JSON +/// object, and `null`, an array or a bare string answered with a success status +/// is an intermediary talking, not a configuration. Neither is a truncated +/// answer, and either can arrive with bytes that are not valid UTF-8 -- a +/// Latin-1 error page does -- or without, so both arms ask the same question. +/// Only storing differs: bytes a Rust string can carry are pretty-printed, and +/// bytes it cannot are written exactly as they arrived. +fn prettify_or_keep(raw: &[u8], path: &str) -> Result, UnitctlError> { + // Show the body along with the parser's complaint: "expected value at line 1 + // column 1" says nothing about the captive portal's sign-in page that + // provoked it, and the page is what tells the operator what happened. + // json_body::body_for_display bounds it and takes its control characters out. + let refuse = |error: serde_json::Error| UnitctlError::DeserializationError { + message: format!("{}: {}\n{}", path, error, json_body::body_for_display(raw)), + }; + + match std::str::from_utf8(raw) { + Ok(text) => serde_json::from_str::(text) + .and_then(|config| serde_json::to_string_pretty(&config)) + .map(String::into_bytes) + .map_err(refuse), + Err(_) => { + // The lossy view answers whether this is a configuration without + // being what gets written: the bytes stored below are the server's + // own. Replacing a byte cannot turn valid JSON into invalid JSON, + // nor an object into something else, so a document that passes this + // way passed before the replacement too. + serde_json::from_str::(&String::from_utf8_lossy(raw)).map_err(refuse)?; + + eprintln!( + "Warning: {} holds bytes that are not valid UTF-8; saving what the server sent, verbatim", + path + ); + eprintln!("Warning: 'unitctl import' cannot replay this file -- restore it with curl"); + Ok(raw.to_vec()) + } + } +} pub async fn cmd(cli: &UnitCtl, filename: &String) -> Result<(), UnitctlError> { - if cli.control_socket_addresses.as_ref().map_or(false, |addrs| addrs.len() > 1) { + if cli + .control_socket_addresses + .as_ref() + .map_or(false, |addrs| addrs.len() > 1) + { return Err(UnitctlError::ControlSocketError { kind: ControlSocketErrorKind::General, message: "too many control sockets. specify at most one.".to_string(), @@ -23,13 +75,7 @@ pub async fn cmd(cli: &UnitCtl, filename: &String) -> Result<(), UnitctlError> { eprintln!("Warning: writing uncompressed tarball to {}", filename); } - let config_res = - serde_json::to_string_pretty(&send_empty_body_deserialize_response(&client, "GET", "/config").await?); - if let Err(e) = config_res { - return Err(UnitctlError::DeserializationError { message: e.to_string() }); - } - - let current_config = config_res.unwrap().into_bytes(); + let current_config = prettify_or_keep(&send_empty_body_read_bytes(&client, "GET", "/config").await?, "/config")?; //let current_js_modules = send_empty_body_deserialize_response(&client, "GET", "/js_modules") // .await?; @@ -53,3 +99,148 @@ pub async fn cmd(cli: &UnitCtl, filename: &String) -> Result<(), UnitctlError> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_configuration_is_pretty_printed() { + let saved = prettify_or_keep(br#"{"listeners":{"*:8080":{"pass":"routes"}}}"#, "/config") + .expect("a JSON body must be saved"); + let text = String::from_utf8(saved).expect("a UTF-8 body stays UTF-8"); + assert!(text.starts_with("{\n"), "expected pretty output, got {:?}", text); + assert!(text.contains("\"*:8080\"")); + } + + /// The one case that keeps the server's bytes: no Rust string can carry + /// them, so re-encoding would change the configuration being backed up. + #[test] + fn bytes_that_are_not_utf8_are_kept_verbatim() { + let mut raw = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + raw.push(0xff); + raw.extend_from_slice(br#""},"action":{"return":204}}]}"#); + + let saved = prettify_or_keep(&raw, "/config").expect("a legacy body must still be saved"); + assert_eq!(saved, raw, "the backup must be the bytes the server sent"); + } + + /// A 200 carrying something that is not a configuration must not be written + /// into the tarball as if it were one. + #[test] + fn a_valid_utf8_body_that_is_not_json_fails() { + let error = prettify_or_keep(b"

502 Bad Gateway

", "/config") + .expect_err("a non-JSON body is not a configuration"); + match error { + UnitctlError::DeserializationError { message } => { + assert!( + message.contains("/config"), + "the message must name the path: {}", + message + ) + } + other => panic!("expected a deserialization error, got {:?}", other), + } + + // Truncated, which is the same failure for a different reason. + assert!(prettify_or_keep(br#"{"listeners":"#, "/config").is_err()); + } + + /// Being unreadable as UTF-8 is not a pass either. A proxy's Latin-1 error + /// page fails both tests at once, and it must fail on the one that matters: + /// it is not a configuration. + #[test] + fn a_body_that_is_neither_utf8_nor_json_fails() { + // "Erreur de passerelle" with the accents written the Latin-1 way, which + // is not valid UTF-8. + let mut page = b"

502 Erreur de passerell".to_vec(); + page.extend_from_slice(&[0xe9, 0xe8]); + page.extend_from_slice(b"

"); + assert!(std::str::from_utf8(&page).is_err(), "the page must not be valid UTF-8"); + + let error = prettify_or_keep(&page, "/config").expect_err("a Latin-1 error page is not a configuration"); + match error { + UnitctlError::DeserializationError { message } => { + assert!( + message.contains("/config"), + "the message must name the path: {}", + message + ) + } + other => panic!("expected a deserialization error, got {:?}", other), + } + + // A document cut short mid-string, holding an undecodable byte as well. + let mut truncated = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + truncated.push(0xff); + assert!(prettify_or_keep(&truncated, "/config").is_err()); + } + + /// Valid JSON is not enough either: a configuration is an object, and a 200 + /// carrying `null` or an array is an intermediary talking. Archiving one + /// would write a `config.json` that cannot be a configuration, and `edit` + /// would offer it for editing and then PUT it back as the whole thing. + #[test] + fn a_body_that_is_not_an_object_fails() { + for body in [&b"null"[..], b"[]", b"[{\"listeners\":{}}]", b"\"a string\"", b"204"] { + let error = prettify_or_keep(body, "/config").expect_err("only an object is a configuration"); + match error { + UnitctlError::DeserializationError { message } => assert!( + message.contains("/config"), + "the message must name the path: {}", + message + ), + other => panic!("expected a deserialization error, got {:?}", other), + } + } + } + + /// The same question on the other arm: being unreadable as UTF-8 must not + /// let a non-object through to the tarball. + #[test] + fn a_non_object_holding_an_undecodable_byte_fails() { + let mut array = br#"["caf"#.to_vec(); + array.push(0xff); + array.extend_from_slice(br#""]"#); + assert!(std::str::from_utf8(&array).is_err(), "the body must not be valid UTF-8"); + assert!(serde_json::from_str::(&String::from_utf8_lossy(&array)).is_ok()); + + assert!( + prettify_or_keep(&array, "/config").is_err(), + "an array is not a configuration, whatever its bytes" + ); + } + + /// The page is what tells the operator what happened; the parser's complaint + /// about its first character does not. Both are shown. + #[test] + fn a_refusal_shows_the_body_it_refused() { + let page = b"captive portal: sign in at http://portal.example/"; + let message = prettify_or_keep(page, "/config") + .expect_err("not a configuration") + .to_string(); + + assert!(message.contains("/config"), "{}", message); + assert!(message.contains("expected value"), "the reason stays: {}", message); + assert!( + message.contains("captive portal: sign in at http://portal.example/"), + "the page must be shown: {}", + message + ); + + // The same holds on the other arm, where the body is not valid UTF-8: + // a Latin-1 page is shown too, with the undecodable bytes as U+FFFD. + let mut latin1 = b"

502 Erreur de passerell".to_vec(); + latin1.extend_from_slice(&[0xe9, 0xe8]); + latin1.extend_from_slice(b"

"); + let message = prettify_or_keep(&latin1, "/config") + .expect_err("not a configuration") + .to_string(); + assert!(message.contains("502 Erreur de passerell"), "{}", message); + assert!( + message.contains('\u{fffd}'), + "undecodable bytes are shown as U+FFFD: {}", + message + ); + } +} diff --git a/tools/unitctl/unitctl/src/inputfile.rs b/tools/unitctl/unitctl/src/inputfile.rs index b2479d505..6f2083de0 100644 --- a/tools/unitctl/unitctl/src/inputfile.rs +++ b/tools/unitctl/unitctl/src/inputfile.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::io; use std::io::{BufRead, BufReader, Error as IoError, Read}; use std::path::{Path, PathBuf}; @@ -161,7 +160,12 @@ impl InputFile { .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })? } InputFormat::Hjson => { - let hjson_value: HashMap = nu_json::from_reader(reader) + // nu_json::Map and not HashMap: with the crate's preserve_order + // feature, on by default, it is a LinkedHashMap and keeps the + // document's member order. A HashMap here shuffled the members + // of every hjson input, which is the same defect the response + // side of this fixes. + let hjson_value: nu_json::Map = nu_json::from_reader(reader) .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?; hjson_value @@ -286,4 +290,111 @@ mod tests { InputFormat::Pem ); } + + /// Member names in an order no map sorts into by accident, so a map that + /// does not keep insertion order cannot pass by luck. + const MEMBERS: [&str; 6] = [ + "settings", + "listeners", + "routes", + "applications", + "access_log", + "upstreams", + ]; + + fn write_input(body: &str, suffix: &str) -> tempfile::NamedTempFile { + let mut file = tempfile::Builder::new() + .prefix("unitctl-inputfile-") + .suffix(suffix) + .tempfile() + .expect("a temporary file"); + std::io::Write::write_all(file.as_file_mut(), body.as_bytes()).expect("writing the input"); + file + } + + fn members_of(file: &tempfile::NamedTempFile, format: InputFormat) -> Vec { + InputFile::FileWithFormat(file.path().into(), format) + .to_unit_serializable_map() + .expect("the input must deserialize") + .keys() + .cloned() + .collect() + } + + /// `UnitSerializableMap` is a `serde_json::Map`, which keeps member order + /// only because every format feeding it keeps it too. `serde_yaml`, `json5` + /// and the hjson `collect()` each reach the type by a different route, so + /// each one is checked. + #[test] + fn every_input_format_keeps_member_order() { + let json = MEMBERS + .iter() + .map(|name| format!(" \"{}\": {{}}", name)) + .collect::>() + .join(",\n"); + let json = format!("{{\n{}\n}}\n", json); + + assert_eq!(members_of(&write_input(&json, ".json"), InputFormat::Json), MEMBERS); + assert_eq!(members_of(&write_input(&json, ".json5"), InputFormat::Json5), MEMBERS); + assert_eq!(members_of(&write_input(&json, ".hjson"), InputFormat::Hjson), MEMBERS); + + let yaml = MEMBERS + .iter() + .map(|name| format!("{}: {{}}\n", name)) + .collect::(); + assert_eq!(members_of(&write_input(&yaml, ".yaml"), InputFormat::Yaml), MEMBERS); + } + + /// The values have to arrive intact, not just the names: hjson reaches + /// `serde_json::Value` through a conversion of its own. + #[test] + fn an_hjson_input_keeps_its_values_and_nesting() { + let file = write_input( + "{\n listeners: {\n \"*:8080\": { pass: \"routes\", tls: { certificate: \"bundle\", session: {} } }\n }\n routes: [ { action: { return: 204 } } ]\n}\n", + ".hjson", + ); + let map = InputFile::FileWithFormat(file.path().into(), InputFormat::Hjson) + .to_unit_serializable_map() + .expect("hjson must deserialize"); + + assert_eq!(map.keys().cloned().collect::>(), vec!["listeners", "routes"]); + assert_eq!(map["listeners"]["*:8080"]["pass"], "routes"); + assert_eq!(map["routes"][0]["action"]["return"], 204); + + // Nested order is the other half of this, and it holds only while both + // crates keep preserve_order: every nu_json::Value goes through + // serde_json::to_value on the way in. So pin the nested names too, in + // an order no map sorts into by accident. + let listener = map["listeners"]["*:8080"].as_object().expect("a listener"); + assert_eq!(listener.keys().cloned().collect::>(), vec!["pass", "tls"]); + let tls = listener["tls"].as_object().expect("a tls object"); + assert_eq!(tls.keys().cloned().collect::>(), vec!["certificate", "session"]); + } + + /// The way back in is strict, which is why `unitctl export` warns that the + /// verbatim backup it writes for a configuration holding bytes that are not + /// valid UTF-8 is one `unitctl import` cannot replay. + #[test] + fn a_file_that_is_not_valid_utf8_is_refused_on_the_way_in() { + let mut file = tempfile::Builder::new() + .suffix(".json") + .tempfile() + .expect("a temporary file"); + let mut raw = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + raw.push(0xff); + raw.extend_from_slice(br#""},"action":{"return":204}}]}"#); + std::io::Write::write_all(file.as_file_mut(), &raw).expect("writing the input"); + + assert!(InputFile::FileWithFormat(file.path().into(), InputFormat::Json) + .to_unit_serializable_map() + .is_err()); + } + + #[test] + fn an_unsupported_input_format_is_refused() { + let file = write_input("not a configuration", ".txt"); + assert!(InputFile::FileWithFormat(file.path().into(), InputFormat::Pem) + .to_unit_serializable_map() + .is_err()); + } } diff --git a/tools/unitctl/unitctl/src/requests.rs b/tools/unitctl/unitctl/src/requests.rs index d56efbd0b..0053c1fc6 100644 --- a/tools/unitctl/unitctl/src/requests.rs +++ b/tools/unitctl/unitctl/src/requests.rs @@ -3,6 +3,8 @@ use super::UnitClient; use super::UnitSerializableMap; use super::UnitctlError; use crate::known_size::KnownSize; +use bytes::Bytes; +use http_body_util::Full; use hyper::Request; use rustls_pemfile::Item; use std::collections::HashMap; @@ -149,6 +151,21 @@ pub async fn send_body_deserialize_response serde::Deserializ .map_err(|e| UnitctlError::UnitClientError { source: e }) } +/// Read a document without decoding it. +/// +/// A caller that will hand the document back to Unit needs the bytes Unit +/// sent: a configuration can hold bytes that no Rust `String` carries, so +/// anything routed through `serde_json` on the way out would return something +/// other than what it was shown. +pub async fn send_empty_body_read_bytes(client: &UnitClient, method: &str, path: &str) -> Result { + let request = build_request(client, method, path, None, KnownSize::Empty) + .map_err(|e| UnitctlError::UnitClientError { source: e })?; + client + .send_request_and_collect_body(request) + .await + .map_err(|e| UnitctlError::UnitClientError { source: e }) +} + async fn streaming_upload_deserialize_response serde::Deserialize<'de>>( client: &UnitClient, method: &str, @@ -156,6 +173,17 @@ async fn streaming_upload_deserialize_response serde::Deseria mime_type: Option, read: KnownSize, ) -> Result { + let request = build_request(client, method, path, mime_type, read)?; + client.send_request_and_deserialize_response(request).await +} + +fn build_request( + client: &UnitClient, + method: &str, + path: &str, + mime_type: Option, + read: KnownSize, +) -> Result>, UnitClientError> { let uri = client.control_socket.create_uri_with_path(path); // Materialize the body first so that Content-Length reflects actual bytes produced, @@ -178,5 +206,5 @@ async fn streaming_upload_deserialize_response serde::Deseria .insert("Content-Type", content_type.parse().unwrap()); } - client.send_request_and_deserialize_response(request).await + Ok(request) } diff --git a/tools/unitctl/unitctl/src/unitctl_error.rs b/tools/unitctl/unitctl/src/unitctl_error.rs index 83b2da465..ea37b2b06 100644 --- a/tools/unitctl/unitctl/src/unitctl_error.rs +++ b/tools/unitctl/unitctl/src/unitctl_error.rs @@ -36,6 +36,11 @@ custom_error! {pub UnitctlError PathNotFound { path: String } = "Path not found: {path}", UnknownInputFileType { path: String } = "Unknown input type for file: {path}", NoFilesImported = "All imports failed", + UndecodableConfiguration { + path: String, + members: String, + values: String, + names: String } = "The configuration at {path} holds bytes that are not valid UTF-8: {members}", WaitTimeoutError = "Timeout waiting for unit to start has been exceeded", } @@ -118,6 +123,32 @@ pub fn eprint_error(error: &UnitctlError) { UnitctlError::UiServerError { ref message } => { eprintln!("UI server error: {}", message); } + UnitctlError::UndecodableConfiguration { + path, values, names, .. + } => { + // The sentence itself is the one custom_error! already defines, so + // that the two cannot drift apart. + eprintln!("{}", error); + eprintln!("Editing it would replace them, so unitctl will not open it."); + eprintln!("Read it with 'unitctl execute -m GET -p {}'.", path); + eprintln!("The bytes appear there as U+FFFD."); + + // A replaced value and a replaced name need different advice. + if !values.is_empty() { + eprintln!(); + eprintln!("These members hold a replaced byte in their value: {}", values); + eprintln!("Repair each one with a PUT to its own path."); + } + + if !names.is_empty() { + eprintln!(); + eprintln!("These are member names, not values: {}", names); + eprintln!("A PUT cannot repair them. The path shown holds U+FFFD where the server"); + eprintln!("holds a byte, so it names a member that does not exist. Get the name as"); + eprintln!("stored from 'unitctl export', which keeps the server's bytes. Then"); + eprintln!("rewrite the nearest parent that decodes: its object, or {}.", path); + } + } _ => { eprintln!("{}", error); } From a8e2700fef08aa6c291c189f819e8b01b004295b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 14:48:07 +0200 Subject: [PATCH 138/391] docs(changes): record the unitctl decoding and ordering fixes Two entries for 1.36.2: unitctl no longer gives up on a configuration it cannot decode as UTF-8, and it no longer reorders the members of one. Both go in the apply="unit" stanza; the per-module stanza is untouched, so the file still carries two ver="1.36.2" blocks and the module packages keep building. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 16 ++++++++++++++++ docs/changes.xml | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/CHANGES b/CHANGES index 0915dc3e2..a5e8d22a4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,22 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: unitctl gave up on a configuration it could not decode as + UTF-8. Unit stores the bytes it is given, so a configuration can + hold bytes no UTF-8 decoder accepts; "unitctl execute" and "unitctl + export" then failed with "JSON error [path=/config]" and left the + operator no way to read or back up the configuration. Both work + now: the document is shown with the undecodable bytes replaced and + the members they fall in named, and "export" stores the server's + bytes verbatim. "unitctl edit" refuses such a configuration rather + than writing the replacements back over it. + + *) Bugfix: unitctl reordered the members of a configuration. "unitctl + edit" and "unitctl export" passed the document through an unordered + map, so members came back in an arbitrary order and an edit rewrote + the whole configuration even where nothing had changed. The order + the server sent is now kept. + *) Change: two source files that the build never used have been removed. src/nxt_job_cache_file.c is named nowhere in auto/, so no configuration ever compiled it, and nothing in the tree referenced diff --git a/docs/changes.xml b/docs/changes.xml index 0e5bce7cd..e6af75463 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -101,6 +101,28 @@ application that relied on them now needs "user": "root".
+ + +unitctl gave up on a configuration it could not decode as UTF-8. Unit stores +the bytes it is given, so a configuration can hold bytes no UTF-8 decoder +accepts; "unitctl execute" and "unitctl export" then failed with "JSON error +[path=/config]" and left the operator no way to read or back up the +configuration. Both work now: the document is shown with the undecodable +bytes replaced and the members they fall in named, and "export" stores the +server's bytes verbatim. "unitctl edit" refuses such a configuration rather +than writing the replacements back over it. + + + + + +unitctl reordered the members of a configuration. "unitctl edit" and "unitctl +export" passed the document through an unordered map, so members came back in +an arbitrary order and an edit rewrote the whole configuration even where +nothing had changed. The order the server sent is now kept. + + + From d44a18e3f5d2b71616c862ac73ca2c666c2b44fa Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 13:40:09 +0200 Subject: [PATCH 139/391] build(java): fail clearly when a jar download or its checksum goes wrong A transient Maven failure currently reds the "clang-ast" job on unrelated pull requests, and the error it prints points at the wrong thing: getting remote tomcat-servlet-api-9.0.121.jar ... ######################################################## 100.0% SHA512 not matched for tomcat-servlet-api-9.0.121.jar, removing ... mv: cannot stat '.../.tomcat-servlet-api-9.0.121.jar.23080': No such file Three separate problems produce that: *) curl ran without --fail, so a 404 or an error page was written out as if it were the jar and curl still exited 0. The checksum then rejected the bytes, which is the right outcome reached the wrong way. *) java_chk_sha512 deletes the file it rejects, and the caller's next statement is "mv" on that file. Under "sh -e" the build aborts on the mv, so the message a reader sees is "cannot stat" rather than anything about a checksum. *) An unpinned jar makes the "grep" for its checksum yield nothing, so the check line is malformed and sha512sum fails. That reports "SHA512 not matched", which is indistinguishable from a corrupt download but means something quite different: the version was bumped without updating auto/modules/java_jar.sha512. Add --fail, --location and --retry to the download, and report each of the three cases distinctly, quoting the URL and the expected checksum. Nothing about which artifacts are fetched or which checksums are enforced changes. The pinned checksum was verified correct while writing this: the entry at auto/modules/java_jar.sha512:12 matches what Maven Central serves today (285900 bytes), so the observed CI failure was a bad transfer, not a changed artifact -- exactly the case --retry is for. Verified by driving auto/modules/java_get_jar directly through five cases: a 404 URL, a 200 carrying wrong content, an unpinned jar, a successful download, and a second run served from the local cache. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- auto/modules/java_chk_sha512 | 16 ++++++++++++++++ auto/modules/java_get_jar | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/auto/modules/java_chk_sha512 b/auto/modules/java_chk_sha512 index 10891cee0..fe259c582 100644 --- a/auto/modules/java_chk_sha512 +++ b/auto/modules/java_chk_sha512 @@ -37,6 +37,22 @@ fi if [ -f "$NXT_JAR_CHK_FILE" ]; then NXT_JAR_SHA512=`grep -F $NXT_JAR_FILE auto/modules/java_jar.sha512 | head -c 128` + + # An unpinned jar produced an empty expected value, a malformed check + # line, and a "SHA512 not matched" that reads exactly like a corrupt + # download. It is neither: it means the version was bumped without + # updating auto/modules/java_jar.sha512. + if [ -z "$NXT_JAR_SHA512" ]; then + $echo + $echo $0: error: no pinned SHA512 for "$NXT_JAR_FILE". + $echo + $echo " Add its checksum to auto/modules/java_jar.sha512;" + $echo " a version bump needs the pinned checksums updated too." + $echo + $echo "error: no pinned SHA512 for $NXT_JAR_FILE" >> $NXT_AUTOCONF_ERR + exit 1 + fi + NXT_JAR_CHK=${NXT_JAR_CHK_FILE}.sha512.$$ $echo "$NXT_JAR_SHA512 $NXT_JAR_CHK_FILE" > $NXT_JAR_CHK diff --git a/auto/modules/java_get_jar b/auto/modules/java_get_jar index 81b300f91..0fcf2f26d 100644 --- a/auto/modules/java_get_jar +++ b/auto/modules/java_get_jar @@ -25,11 +25,44 @@ if [ ! -f "$NXT_BUILD_DIR/$NXT_JAR_FILE" ]; then $echo "getting remote $NXT_JAR_FILE ..." >> $NXT_AUTOCONF_ERR mkdir -p "${NXT_JAR_LOCAL_DIR}" - curl --progress-bar "$NXT_JAR_URL" -o "$NXT_JAR_LOCAL_TMP" + + # --fail so an error page is not written out as if it were the jar, + # --location to follow a repository redirect, and --retry because a + # transient failure here fails the whole build for reasons that have + # nothing to do with the change under test. + if ! curl --fail --location --retry 3 --retry-delay 2 \ + --progress-bar "$NXT_JAR_URL" -o "$NXT_JAR_LOCAL_TMP" + then + rm -f "$NXT_JAR_LOCAL_TMP" + $echo + $echo $0: error: failed to download "$NXT_JAR_URL". + $echo + $echo "error: failed to download $NXT_JAR_URL" >> $NXT_AUTOCONF_ERR + exit 1 + fi NXT_JAR_CHK_FILE=$NXT_JAR_LOCAL_TMP . auto/modules/java_chk_sha512 + # java_chk_sha512 deletes the file it rejects, so a missing temp file + # here means the checksum did not match. Say that, rather than + # letting the mv below fail with a "cannot stat" that hides the cause. + if [ ! -f "$NXT_JAR_LOCAL_TMP" ]; then + $echo + $echo $0: error: SHA512 mismatch for "$NXT_JAR_FILE". + $echo + $echo " downloaded from: $NXT_JAR_URL" + $echo " expected SHA512: $NXT_JAR_SHA512" + $echo + $echo " The pinned checksum is in auto/modules/java_jar.sha512." + $echo " A mismatch means the download was truncated or proxied," + $echo " or the artifact itself changed." + $echo + $echo "error: SHA512 mismatch for $NXT_JAR_FILE" \ + >> $NXT_AUTOCONF_ERR + exit 1 + fi + mv "$NXT_JAR_LOCAL_TMP" "$NXT_JAR_LOCAL" else $echo "getting local $NXT_JAR_FILE" From 5e51d937a2d05fcd7e340c2b04b457cff6fe092d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 14:07:54 +0200 Subject: [PATCH 140/391] build(java): retry the verification, not just the transfer The previous commit claimed --retry covered the failure seen in CI. It does not. curl retries timeouts and 5xx; the observed failure is a COMPLETE transfer of the wrong bytes -- a "100.0%" progress bar in 0.07 seconds for a 280 KB jar, then a checksum mismatch. As far as curl is concerned that transfer succeeded, so no curl-level retry setting, --retry-all-errors included, would try again. Wrap the download and its verification in one bounded loop instead, so a poisoned cache entry or an error page served with a 200 is retried rather than failing the build. Three attempts. Keep the two causes apart in the final message: with the retry loop, a missing temp file no longer implies a checksum failure -- a download that never arrived leaves the same absence -- so track which one exhausted the attempts and report that. Otherwise a 404 would be reported as a checksum mismatch, which is how this file came to be confusing in the first place. Verified through the same four cases as before: a 404 URL (three retries, then "failed to download"), a 200 carrying wrong content (three retries, then "SHA512 mismatch"), an unpinned jar (no retry, since retrying cannot supply a checksum nobody wrote down), and a real download. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- auto/modules/java_get_jar | 56 ++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/auto/modules/java_get_jar b/auto/modules/java_get_jar index 0fcf2f26d..ad2348d1e 100644 --- a/auto/modules/java_get_jar +++ b/auto/modules/java_get_jar @@ -26,14 +26,50 @@ if [ ! -f "$NXT_BUILD_DIR/$NXT_JAR_FILE" ]; then mkdir -p "${NXT_JAR_LOCAL_DIR}" - # --fail so an error page is not written out as if it were the jar, - # --location to follow a repository redirect, and --retry because a - # transient failure here fails the whole build for reasons that have - # nothing to do with the change under test. - if ! curl --fail --location --retry 3 --retry-delay 2 \ - --progress-bar "$NXT_JAR_URL" -o "$NXT_JAR_LOCAL_TMP" + # The observed CI failure is a COMPLETE transfer of the wrong bytes: + # a "100.0%" progress bar in 0.07s for a 280 KB jar, then a checksum + # mismatch -- an error page or a poisoned cache, not a truncated + # download. curl's own --retry does not cover that, since as far as + # curl is concerned the transfer succeeded, so the retry has to span + # the verification too. --fail keeps an error page from being written + # out as if it were the jar, and --location follows a redirect. + NXT_JAR_TRY=1 + NXT_JAR_FAIL=download + + while [ $NXT_JAR_TRY -le 3 ]; do + + if curl --fail --location --retry 2 --retry-delay 2 \ + --progress-bar "$NXT_JAR_URL" -o "$NXT_JAR_LOCAL_TMP" + then + NXT_JAR_CHK_FILE=$NXT_JAR_LOCAL_TMP + . auto/modules/java_chk_sha512 + + # java_chk_sha512 deletes the file it rejects, so the file + # surviving means the checksum matched. + if [ -f "$NXT_JAR_LOCAL_TMP" ]; then + break + fi + + NXT_JAR_FAIL=checksum + + $echo "retrying $NXT_JAR_FILE: checksum did not match" + else + rm -f "$NXT_JAR_LOCAL_TMP" + + NXT_JAR_FAIL=download + + $echo "retrying $NXT_JAR_FILE: download failed" + fi + + NXT_JAR_TRY=$((NXT_JAR_TRY + 1)) + done + + # Every attempt was rejected. Say why, rather than letting the mv + # below fail with a "cannot stat" that hides the cause, and keep the + # two causes apart: a download that never arrived is a different + # problem from bytes that arrived and did not verify. + if [ ! -f "$NXT_JAR_LOCAL_TMP" ] && [ "$NXT_JAR_FAIL" = "download" ] then - rm -f "$NXT_JAR_LOCAL_TMP" $echo $echo $0: error: failed to download "$NXT_JAR_URL". $echo @@ -41,12 +77,6 @@ if [ ! -f "$NXT_BUILD_DIR/$NXT_JAR_FILE" ]; then exit 1 fi - NXT_JAR_CHK_FILE=$NXT_JAR_LOCAL_TMP - . auto/modules/java_chk_sha512 - - # java_chk_sha512 deletes the file it rejects, so a missing temp file - # here means the checksum did not match. Say that, rather than - # letting the mv below fail with a "cannot stat" that hides the cause. if [ ! -f "$NXT_JAR_LOCAL_TMP" ]; then $echo $echo $0: error: SHA512 mismatch for "$NXT_JAR_FILE". From f318ff5eb2c13db0c1a8df06cf7118c5938e3f1e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 16:33:24 +0200 Subject: [PATCH 141/391] unitctl: read PEM sections with rustls-pki-types rustls-pemfile is unmaintained (RUSTSEC-2025-0134). Its 2.x releases are a thin layer over rustls-pki-types, and that crate is already in the graph. So this removes a crate without adding one. unitctl uses the parser in one place. Before a PEM upload is sent, it reads the file and refuses it unless the file holds at least one certificate and at least one private key. That check is unchanged. The old parser turned each section into an Item and skipped the kinds it had no Item for. The new code reads the section kind directly and drops the kinds this check does not name. That is the same set of kinds. A parse failure is still reported as an IO error. rustls-pemfile built that error itself, and pki-types builds its own, so for a malformed file the wording after "IO error: " changes a little. The "std" feature of rustls-pki-types is not on by default, and pem::from_buf needs it, so the dependency asks for it. The stale "extern crate rustls_pemfile" in main.rs goes too; edition 2021 does not need those lines. The old and new code were compared rather than assumed equal. A throwaway test ran both over 1884 PEM documents. Those were every one, two and three section combination of eight section kinds, plus junk, empty, truncated and bad-base64 input. All 1884 agreed. Making PublicKey count as a private key made 167 of them disagree, which shows the comparison could see a difference. That test needs rustls-pemfile, so it is not in this commit. The eight tests that are here cover the same behaviour. 175 crates before, 174 after. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/Cargo.lock | 11 +- tools/unitctl/unitctl/Cargo.toml | 3 +- tools/unitctl/unitctl/src/main.rs | 1 - tools/unitctl/unitctl/src/requests.rs | 157 +++++++++++++++++++++++--- 4 files changed, 142 insertions(+), 30 deletions(-) diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 5727c5a91..2a5e67ebe 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -1395,15 +1395,6 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.15.0" @@ -1884,7 +1875,7 @@ dependencies = [ "hyper", "json5", "nu-json", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_yaml", diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index 67a64c6aa..5da03a182 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -20,7 +20,8 @@ json5 = "1.3.1" nu-json = "0.113.0" serde_json = { version = "1.0", optional = false } serde_yaml = "0.9" -rustls-pemfile = "2.0.0" +# The "std" feature is not on by default and pem::from_buf needs it. +rustls-pki-types = { version = "1.15", features = ["std"] } unit-client-rs = { path = "../unit-client-rs" } colored_json = "5.0.0" tempfile = "3.8" diff --git a/tools/unitctl/unitctl/src/main.rs b/tools/unitctl/unitctl/src/main.rs index 7e054d015..b7a779fde 100644 --- a/tools/unitctl/unitctl/src/main.rs +++ b/tools/unitctl/unitctl/src/main.rs @@ -2,7 +2,6 @@ extern crate clap; extern crate colored_json; extern crate custom_error; extern crate nu_json; -extern crate rustls_pemfile; extern crate serde; extern crate unit_client_rs; diff --git a/tools/unitctl/unitctl/src/requests.rs b/tools/unitctl/unitctl/src/requests.rs index 0053c1fc6..1317b9d0b 100644 --- a/tools/unitctl/unitctl/src/requests.rs +++ b/tools/unitctl/unitctl/src/requests.rs @@ -6,9 +6,12 @@ use crate::known_size::KnownSize; use bytes::Bytes; use http_body_util::Full; use hyper::Request; -use rustls_pemfile::Item; +use rustls_pki_types::pem; +use rustls_pki_types::pem::SectionKind; use std::collections::HashMap; +use std::io::BufRead; use std::io::Cursor; +use std::io::ErrorKind; use std::sync::atomic::AtomicUsize; use unit_client_rs::unit_client::UnitClientError; @@ -60,10 +63,7 @@ pub async fn send_and_validate_pem_data_deserialize_response( let bytes: Vec = input_file.try_into()?; { let mut cursor = Cursor::new(&bytes); - let items = rustls_pemfile::read_all(&mut cursor) - .map(|item| item.map_err(|e| UnitctlError::IoError { source: e })) - .collect(); - validate_pem_items(items)?; + validate_pem_items(read_pem_sections(&mut cursor))?; } let known_size = KnownSize::Vec((*bytes).to_owned()); @@ -72,21 +72,61 @@ pub async fn send_and_validate_pem_data_deserialize_response( .map_err(|e| UnitctlError::UnitClientError { source: e }) } -/// Validate the contents of a PEM file -fn validate_pem_items(pem_items: Vec>) -> Result<(), UnitctlError> { - fn item_name(item: Item) -> String { - match item { - Item::X509Certificate(_) => "X509Certificate", - Item::Sec1Key(_) => "Sec1Key", - Item::Crl(_) => "Crl", - Item::Pkcs1Key(_) => "Pkcs1Key", - Item::Pkcs8Key(_) => "Pkcs8Key", - // Note: this is not a valid PEM item, but rustls_pemfile library defines the enum as non-exhaustive - _ => "Unknown", +/// Name a PEM section for the tally below. +/// +/// `None` means the section is not one this check knows, and the caller drops +/// it. Only the names ending in "Key" or "Certificate" are counted, so the +/// other names are there to keep the section in the tally. +fn section_name(kind: SectionKind) -> Option<&'static str> { + match kind { + SectionKind::Certificate => Some("X509Certificate"), + SectionKind::EcPrivateKey => Some("Sec1Key"), + SectionKind::Crl => Some("Crl"), + SectionKind::RsaPrivateKey => Some("Pkcs1Key"), + SectionKind::PrivateKey => Some("Pkcs8Key"), + SectionKind::PublicKey => Some("Unknown"), + SectionKind::Csr => Some("Unknown"), + // SectionKind is non-exhaustive. A kind we do not name is dropped, so + // a file that carries one is judged on its other sections alone. + _ => None, + } +} + +/// Read every PEM section this check knows about. +/// +/// Reading stops at the first error. The caller reports that error, so +/// anything after it would not be looked at. +fn read_pem_sections(rd: &mut dyn BufRead) -> Vec> { + let mut sections = Vec::new(); + + loop { + match pem::from_buf(rd) { + Ok(Some((kind, _data))) => { + if section_name(kind).is_some() { + sections.push(Ok(kind)); + } + } + Ok(None) => return sections, + Err(error) => { + sections.push(Err(pem_error(error))); + return sections; + } } - .to_string() } +} +/// Turn a PEM parse failure into the error unitctl reports. +fn pem_error(error: pem::Error) -> UnitctlError { + match error { + pem::Error::Io(source) => UnitctlError::IoError { source }, + other => UnitctlError::IoError { + source: std::io::Error::new(ErrorKind::InvalidData, other.to_string()), + }, + } +} + +/// Validate the contents of a PEM file +fn validate_pem_items(pem_items: Vec>) -> Result<(), UnitctlError> { if pem_items.is_empty() { let error = UnitctlError::CertificateError { message: "No certificates found in file".to_string(), @@ -98,7 +138,7 @@ fn validate_pem_items(pem_items: Vec>) -> Result<(), for pem_item_result in pem_items { let pem_item = pem_item_result?; - let key = item_name(pem_item); + let key = section_name(pem_item).unwrap_or("Unknown").to_string(); if let Some(count) = items_tally.get_mut(key.clone().as_str()) { count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); } else { @@ -208,3 +248,84 @@ fn build_request( Ok(request) } + +#[cfg(test)] +mod tests { + use super::*; + + const CERT: &str = "-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n"; + const PKCS8_KEY: &str = "-----BEGIN PRIVATE KEY-----\nAQID\n-----END PRIVATE KEY-----\n"; + const RSA_KEY: &str = "-----BEGIN RSA PRIVATE KEY-----\nAQID\n-----END RSA PRIVATE KEY-----\n"; + const EC_KEY: &str = "-----BEGIN EC PRIVATE KEY-----\nAQID\n-----END EC PRIVATE KEY-----\n"; + const PUBLIC_KEY: &str = "-----BEGIN PUBLIC KEY-----\nAQID\n-----END PUBLIC KEY-----\n"; + const ECH_CONFIG: &str = "-----BEGIN ECHCONFIG-----\nAQID\n-----END ECHCONFIG-----\n"; + + /// Run the check a PEM upload goes through before it is sent. + fn validate(pem: &str) -> Result<(), UnitctlError> { + let mut cursor = Cursor::new(pem.as_bytes()); + validate_pem_items(read_pem_sections(&mut cursor)) + } + + fn error_message(pem: &str) -> String { + match validate(pem) { + Err(UnitctlError::CertificateError { message }) => message, + other => panic!("expected a certificate error, got {:?}", other.is_ok()), + } + } + + #[test] + fn accepts_a_certificate_with_a_key() { + validate(&format!("{}{}", CERT, PKCS8_KEY)).unwrap(); + validate(&format!("{}{}", CERT, RSA_KEY)).unwrap(); + validate(&format!("{}{}", CERT, EC_KEY)).unwrap(); + } + + #[test] + fn accepts_a_chain_of_certificates_with_one_key() { + validate(&format!("{}{}{}", CERT, CERT, PKCS8_KEY)).unwrap(); + } + + #[test] + fn rejects_a_certificate_with_no_key() { + assert_eq!(error_message(CERT), "No private keys found in file"); + } + + #[test] + fn rejects_a_key_with_no_certificate() { + assert_eq!(error_message(PKCS8_KEY), "No certificates found in file"); + } + + #[test] + fn rejects_a_file_with_no_pem_section() { + assert_eq!(error_message(""), "No certificates found in file"); + assert_eq!(error_message("not a pem file\n"), "No certificates found in file"); + } + + /// A public key is not a private key, so it cannot stand in for one. + #[test] + fn rejects_a_public_key_as_the_key() { + assert_eq!( + error_message(&format!("{}{}", CERT, PUBLIC_KEY)), + "No private keys found in file" + ); + } + + /// A section this check does not name is dropped, so the file is judged on + /// its other sections. + #[test] + fn ignores_a_section_it_does_not_name() { + validate(&format!("{}{}{}", CERT, ECH_CONFIG, PKCS8_KEY)).unwrap(); + assert_eq!(error_message(ECH_CONFIG), "No certificates found in file"); + } + + #[test] + fn rejects_a_section_with_no_end_marker() { + let truncated = "-----BEGIN CERTIFICATE-----\nAQID\n"; + match validate(truncated) { + Err(UnitctlError::IoError { source }) => { + assert_eq!(source.kind(), ErrorKind::InvalidData); + } + other => panic!("expected an IO error, got ok={}", other.is_ok()), + } + } +} From f1a65831c4de6a184b932623c9f6801538bb173d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 16:29:42 +0200 Subject: [PATCH 142/391] unitctl: drop HTTP/2 from the control API client The control API speaks HTTP/1.1 only. The controller writes "HTTP/1.1 " into every response line (src/nxt_controller.c:2850). The HTTP/2 code we linked in could never run. It still cost us something. It pulled in h2 and fnv, and h2 0.4.15 carries RUSTSEC-2026-0258. Removing the feature closes that advisory with no version upgrade. Every "http2" here was ours and explicit, so this is a plain removal. hyper-rustls does not enable HTTP/2 in its default features. The crypto provider does not come from its feature list either: unit-client-rs depends on rustls directly with default features, and that is what selects aws_lc_rs. aws-lc-sys is still in the graph after this change and ring is still absent. One code change was needed. enable_all_versions() is behind the http2 feature, and it also offered "h2" in the ALPN list. enable_http1() replaces it. 175 crates before, 173 after. Tests and clippy warnings are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/Cargo.lock | 26 ------------------- tools/unitctl/unit-client-rs/Cargo.toml | 6 ++--- .../unitctl/unit-client-rs/src/unit_client.rs | 4 ++- tools/unitctl/unitctl/Cargo.toml | 2 +- 4 files changed, 7 insertions(+), 31 deletions(-) diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 2a5e67ebe..4e00da7cc 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -451,12 +451,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -594,25 +588,6 @@ dependencies = [ "r-efi 6.0.0", ] -[[package]] -name = "h2" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -686,7 +661,6 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", diff --git a/tools/unitctl/unit-client-rs/Cargo.toml b/tools/unitctl/unit-client-rs/Cargo.toml index a430dac2e..2afc6cca2 100644 --- a/tools/unitctl/unit-client-rs/Cargo.toml +++ b/tools/unitctl/unit-client-rs/Cargo.toml @@ -10,9 +10,9 @@ name = "unit_client_rs" [dependencies] custom_error = "1.9" -hyper = { version = "1", features = ["http1", "http2"] } -hyper-rustls = { version = "0.27", features = ["rustls-native-certs", "webpki-roots", "http2"] } -hyper-util = { version = "0.1", features = ["client-legacy", "client", "http1", "http2", "tokio"] } +hyper = { version = "1", features = ["http1"] } +hyper-rustls = { version = "0.27", features = ["rustls-native-certs", "webpki-roots"] } +hyper-util = { version = "0.1", features = ["client-legacy", "client", "http1", "tokio"] } hyperlocal = "0.9" http-body-util = "0.1" bytes = "1" diff --git a/tools/unitctl/unit-client-rs/src/unit_client.rs b/tools/unitctl/unit-client-rs/src/unit_client.rs index 8a908107c..1b1a5c6e6 100644 --- a/tools/unitctl/unit-client-rs/src/unit_client.rs +++ b/tools/unitctl/unit-client-rs/src/unit_client.rs @@ -241,7 +241,9 @@ impl UnitClient { .with_native_roots() .unwrap_or_else(|_| HttpsConnectorBuilder::new().with_webpki_roots()) .https_or_http() - .enable_all_versions() + // The control API speaks HTTP/1.1 only. This also keeps "h2" out + // of the ALPN list we offer. + .enable_http1() .build(); let remote_client: Client, Full> = Client::builder(TokioExecutor::new()).build(connector); diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index 5da03a182..0f00592de 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -28,7 +28,7 @@ tempfile = "3.8" which = "8.0.2" walkdir = "2.4" -hyper = { version = "1.9.0", features = ["http2", "server", "client"] } +hyper = { version = "1.9.0", features = ["server", "client"] } http-body-util = "0.1" bytes = "1" tokio = { version = "1.35", features = ["macros", "rt-multi-thread"] } From 451495846e8dd645714a35cae4d755114960c640 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 16:29:52 +0200 Subject: [PATCH 143/391] ci(unitctl): audit Cargo.lock against the RustSec database Nobody was checking the lock file. Two advisories were live in it when this job was written, and neither had been noticed. The job reads Cargo.lock only. It needs no openapi generation and no build, so it is fast and does not depend on the other jobs. cargo-audit comes from taiki-e/install-action as a prebuilt binary. That is the pattern the repo already uses for cross and for cargo-component. Two details decide whether this gate works at all. It runs with --deny warnings. A bare "cargo audit" exits 0 on an unmaintained crate, so the job would report success while an advisory was open. It also runs on a weekly schedule. The other triggers are a push and a path-gated pull request, so a new advisory published against a lock file nobody has touched would never start the job. The test and build jobs are excluded from that trigger, because the weekly run is for the audit alone. This comes after the HTTP/2 removal on purpose. Added before it, the job would fail on RUSTSEC-2026-0258, which that commit closes. A gate that is red on the day it lands gets ignored or switched off. With --deny warnings the job is still red on RUSTSEC-2025-0134, the unmaintained warning for rustls-pemfile. The commit that drops rustls-pemfile has to land before this one is green. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/unitctl.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/unitctl.yml b/.github/workflows/unitctl.yml index 1c74c2391..7ca25cc03 100644 --- a/.github/workflows/unitctl.yml +++ b/.github/workflows/unitctl.yml @@ -11,6 +11,10 @@ on: - master tags: - '[0-9]+.[0-9]+.[0-9]+' + schedule: + # The audit job needs a trigger that does not depend on a code change. A + # new advisory can land against a lock file nobody has touched. + - cron: "0 6 * * 1" workflow_dispatch: inputs: version: @@ -31,6 +35,8 @@ concurrency: jobs: test: + # The weekly trigger exists for the audit job alone. + if: github.event_name != 'schedule' runs-on: ${{ matrix.os }} timeout-minutes: 25 defaults: @@ -81,7 +87,34 @@ jobs: - name: Test ${{ matrix.os }} run: ${{ env.MAKE }} test + audit: + # Check Cargo.lock against the RustSec advisory database. This reads the + # lock file only, so it needs no openapi generation and no build. + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: tools/unitctl + steps: + - uses: actions/checkout@v5 + + - run: rustup update stable + + # Prebuilt binary instead of `cargo install cargo-audit`, which would + # compile it from source on every run. + - name: Install cargo-audit + uses: taiki-e/install-action@v2 + with: + tool: cargo-audit + + # Without --deny warnings the command exits 0 on an unmaintained crate, + # so the job would pass while an advisory is open. + - name: Audit Cargo.lock + run: cargo audit --deny warnings + build: + # The weekly trigger exists for the audit job alone. + if: github.event_name != 'schedule' runs-on: ${{ matrix.os }} timeout-minutes: 30 defaults: From 8414907728c4b77e9ca5c926e74af4880b9d39ea Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 16:52:17 +0200 Subject: [PATCH 144/391] docs(changes): record unitctl's rustls-pki-types and http2 removals #344 replaces rustls-pemfile with rustls-pki-types (RUSTSEC-2025-0134); #345 drops hyper's "http2" feature, removing h2 and fnv (RUSTSEC-2026-0258). The control API is HTTP/1.1 only, so the feature was never usable against the server -- but unitctl also stops offering HTTP/2 in the ALPN list for any remote (TLS) control API connection, including through a proxy in front of it. That is a separate Change entry from the Security one. One entry pair on this branch, since it merges after #344 and both changes are present in master by the time this lands. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 17 +++++++++++++++++ docs/changes.xml | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/CHANGES b/CHANGES index a5e8d22a4..679609160 100644 --- a/CHANGES +++ b/CHANGES @@ -89,6 +89,23 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 use of them is finished before the drop, but an application that relied on them now needs "user": "root". + *) Security: unitctl no longer depends on rustls-pemfile or on hyper's + "http2" feature. rustls-pemfile is unmaintained; the parser is replaced + by rustls-pki-types, already in the dependency graph. PEM parsing + behavior is unchanged: both parsers were run over 1884 PEM documents and + produced zero differences. This closes RUSTSEC-2025-0134. Dropping the + "http2" feature from hyper, hyper-rustls and hyper-util removes the h2 + and fnv crates as well, closing RUSTSEC-2026-0258, an unbounded empty + DATA frame flood in h2 0.4.15. The control API speaks HTTP/1.1 only -- + unitd writes "HTTP/1.1" into every response line -- so HTTP/2 support + was never usable against the server itself. + + *) Change: unitctl no longer offers HTTP/2 in the TLS handshake it makes to + a remote control API. This follows from the "http2" feature removal + above. A TLS-terminating proxy placed in front of a remote control API + must now speak HTTP/1.1 to unitctl; one that only accepts HTTP/2 can no + longer be reached. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index e6af75463..e00835356 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -123,6 +123,29 @@ nothing had changed. The order the server sent is now kept.
+ + +unitctl no longer depends on rustls-pemfile or on hyper's "http2" feature. +rustls-pemfile is unmaintained; the parser is replaced by rustls-pki-types, +already in the dependency graph. PEM parsing behavior is unchanged: both +parsers were run over 1884 PEM documents and produced zero differences. This +closes RUSTSEC-2025-0134. Dropping the "http2" feature from hyper, hyper-rustls +and hyper-util removes the h2 and fnv crates as well, closing +RUSTSEC-2026-0258, an unbounded empty DATA frame flood in h2 0.4.15. The +control API speaks HTTP/1.1 only -- unitd writes "HTTP/1.1" into every response +line -- so HTTP/2 support was never usable against the server itself. + + + + + +unitctl no longer offers HTTP/2 in the TLS handshake it makes to a remote +control API. This follows from the "http2" feature removal above. A TLS- +terminating proxy placed in front of a remote control API must now speak +HTTP/1.1 to unitctl; one that only accepts HTTP/2 can no longer be reached. + + + From a8018bc88442d9905c5d75fee425cb8aebc96507 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 14:12:37 +0200 Subject: [PATCH 145/391] port: close the descriptors a handler does not take A descriptor attached to a port message is leaked whenever nothing takes ownership of it: the dispatcher reclaimed nothing a handler left behind, and several paths never reach a handler at all. The reach is wider than the router's own application ports. An application is given the write end of the router's MAIN port -- nxt_application.c sets init->router_port.out_fd to that port's pair[1] -- and nxt_port_handler() dispatches on the type byte with no sender check, so every handler in both router tables is reachable from an application. Among them GET_MMAP and OOSM, which libunit sends as a matter of course and which never look at msg->fd. The 16-byte header is entirely attacker-chosen, type, stream and the nf/mf fragment bits included. The fragment paths run no handler: *) "goto fmsg_failed" skips dispatch when a fragment cannot be found or started, leaving msg->fd untouched. *) A middle fragment is chained into the assembled message without its fd[] being carried across. *) The last-fragment merge overwrote the fragment's own fd[] with the assembled message's. So close at the tail of nxt_port_read_msg_process(), which covers the handler and fragment paths alike, plus one close in the merge before the overwrite. This relies on the convention that a handler keeping a descriptor sets its slot to -1 -- already followed by the port and main-process core, and now followed everywhere. Seven sites did not: three kept a descriptor without clearing (the listen socket and TLS chain file in nxt_router.c, the log descriptor in nxt_router_access_log.c) and four closed one without clearing (two in nxt_controller.c, one each in nxt_router.c and nxt_router_access_log.c). Neither class fails as a leak, which is why both matter. A kept descriptor that is centrally closed is aliased by the next open() in that thread, so the retained handle silently refers to something else; a closed one is closed twice. Getting this wrong shut the listening socket, reported by the test suite as epoll_ctl(11, 1, 51) failed (9: Bad file descriptor) and missing the access-log close produced close(53) failed (9: Bad file descriptor) on log reopen -- found by exercising SIGUSR1 by hand, since no test covers it. Both are fixed here and verified absent: three reopen cycles log correctly with no alerts. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- CHANGES | 11 +++++++++++ docs/changes.xml | 14 +++++++++++++ src/nxt_controller.c | 2 ++ src/nxt_port.h | 18 +++++++++++------ src/nxt_port_rpc.c | 9 +++++++++ src/nxt_port_socket.c | 39 +++++++++++++++++++++++++++++++++++++ src/nxt_router.c | 17 ++++++++++++++++ src/nxt_router_access_log.c | 4 ++++ 8 files changed, 108 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index 679609160..c87c94d42 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,17 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Security: a file descriptor sent to a port is no longer leaked when + the message is dispatched and nothing takes ownership of it. An + application holds the write end of the router's main port and the + message header is unauthenticated, so a compromised application could + attach a descriptor to messages the router does not expect one on -- + or to a fragment that never reaches a handler -- and consume the + router's descriptor table, stopping every application behind it. A + fragment stream that is started and never completed still retains its + descriptor; that needs a bound on pending fragments and is tracked + in freeunitorg/freeunit#343. + *) Bugfix: unitctl gave up on a configuration it could not decode as UTF-8. Unit stores the bytes it is given, so a configuration can hold bytes no UTF-8 decoder accepts; "unitctl execute" and "unitctl diff --git a/docs/changes.xml b/docs/changes.xml index e00835356..eb49ac41f 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,20 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +a file descriptor sent to a port is no longer leaked when the message is +dispatched and nothing takes ownership of it. An application holds the write +end of the router's main port and the message header is unauthenticated, so a +compromised application could attach a descriptor to messages the router does +not expect one on -- or to a fragment that never reaches a handler -- and +consume the router's descriptor table, stopping every application behind it. +A fragment stream that is started and never completed still retains its +descriptor; that needs a bound on pending fragments and is tracked in +freeunitorg/freeunit#343. + + + two source files that the build never used have been removed. diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 690ba6185..ce6dff6e0 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -2069,6 +2069,7 @@ nxt_controller_process_cert_save(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_fd_write(msg->fd[0], mbuf->pos, nxt_buf_mem_used_size(mbuf)); nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; nxt_memzero(&resp, sizeof(nxt_controller_response_t)); @@ -2352,6 +2353,7 @@ nxt_controller_process_script_save(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_fd_write(msg->fd[0], mbuf->pos, nxt_buf_mem_used_size(mbuf)); nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; nxt_memzero(&resp, sizeof(nxt_controller_response_t)); diff --git a/src/nxt_port.h b/src/nxt_port.h index 0d1d1c722..5d5d1c530 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -254,12 +254,18 @@ struct nxt_port_recv_msg_s { /* * Close any file descriptors the peer attached to a received message via - * SCM_RIGHTS. A privileged handler that rejects a message (unauthorized - * or malformed sender) must call this before returning: the port - * dispatcher does not reclaim descriptors once the handler returns, and - * a compromised peer can attach fds to a forged message, so leaving them - * open on the reject path would let it exhaust the receiver's descriptor - * table. + * SCM_RIGHTS. + * + * The ownership contract: nxt_port_read_msg_process() closes whatever is + * left in msg->fd[] once the message has been dispatched, so a handler that + * KEEPS a descriptor must set its slot to -1. A kept descriptor whose slot + * was left set is closed under the handler, and the number is then handed + * out again by the next open() or accept() on that thread -- the retained + * handle silently refers to something else, which is a good deal worse than + * the leak this arrangement replaced. + * + * Calling this explicitly is still right on a reject path, where it makes + * the intent local and obvious, and it is idempotent. */ nxt_inline void nxt_port_recv_msg_close_fds(nxt_port_recv_msg_t *msg) diff --git a/src/nxt_port_rpc.c b/src/nxt_port_rpc.c index 0cdd796fa..1ddf9198c 100644 --- a/src/nxt_port_rpc.c +++ b/src/nxt_port_rpc.c @@ -426,6 +426,15 @@ nxt_port_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (ret != NXT_OK) { nxt_debug(task, "rpc: stream #%uD no handler found", stream); + /* + * No handler ran, so nothing can have taken the descriptors this + * message arrived with. The dispatcher would close them anyway; + * doing it here keeps the reject path self-evidently safe, since + * the stream number is a value off the wire and any peer can name + * one that was never registered. + */ + nxt_port_recv_msg_close_fds(msg); + return; } diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 7a4f868fb..89bc3e970 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -1251,6 +1251,15 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) if (n == (ssize_t) sizeof(nxt_port_msg_t) && msg.port_msg.type == _NXT_PORT_MSG_READ_QUEUE) { + /* + * A wake-up marker carries no payload and needs no + * descriptor, but the sender chooses the type byte and + * SCM_RIGHTS is already attached by the time it is read. + * This exit never reaches nxt_port_read_msg_process(), so + * the close at its tail cannot cover it. + */ + nxt_port_close_fds(msg.fd); + nxt_port_buf_free(port, b); nxt_debug(task, "port{%d,%d} %d: recv %d read_queue", @@ -1282,6 +1291,8 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) (int) port->pid, (int) port->id, port->socket.fd); + nxt_port_close_fds(msg.fd); + return; } @@ -1294,6 +1305,8 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) (int) port->pid, (int) port->id, port->socket.fd); + nxt_port_close_fds(msg.fd); + return; } } @@ -1544,6 +1557,12 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, port->handler(task, fmsg); + /* + * The assembled message's descriptors are about to overwrite + * this fragment's own, which would drop them unnoticed. + */ + nxt_port_close_fds(msg->fd); + msg->buf = fmsg->buf; msg->fd[0] = fmsg->fd[0]; msg->fd[1] = fmsg->fd[1]; @@ -1625,6 +1644,26 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, /* restore original buf */ msg->buf = orig_b; } + + /* + * Close whatever nothing took ownership of. A handler that keeps a + * descriptor sets its slot to -1, so this closes only what is genuinely + * unclaimed, and it is idempotent for the paths that already closed. + * + * This sits at the tail rather than beside the handler call because the + * paths that drop descriptors are not all handler paths: "goto + * fmsg_failed" above skips the dispatch entirely when a fragment cannot + * be found or started, and a middle fragment is chained into the + * assembled message without its own fd[] being carried across. Neither + * runs a handler, so no per-handler close can reach them. + * + * The message header is attacker-chosen -- type, stream and the nf/mf + * fragment bits are bytes off the wire, and an application holds the + * write end of the router's main port -- so every one of those paths is + * reachable from a compromised application, one leaked descriptor per + * message until the table is exhausted. + */ + nxt_port_close_fds(msg->fd); } diff --git a/src/nxt_router.c b/src/nxt_router.c index 210d3c8e1..0bf43ff7b 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -3173,6 +3173,7 @@ nxt_router_js_module_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, ret = nxt_script_file_read(msg->fd[0], &text); nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; if (nxt_slow_path(ret == NXT_ERROR)) { goto fail; @@ -3804,6 +3805,9 @@ nxt_router_listen_socket_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, s = msg->fd[0]; + /* The listener owns the descriptor now. */ + msg->fd[0] = -1; + ret = nxt_socket_nonblocking(task, s); if (nxt_slow_path(ret != NXT_OK)) { goto fail; @@ -3941,6 +3945,10 @@ nxt_router_tls_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, } bundle->chain_file = msg->fd[0]; + + /* The bundle owns the descriptor now. */ + msg->fd[0] = -1; + bundle->next = tlscf->bundle; tlscf->bundle = bundle; @@ -5252,6 +5260,15 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_unit_response_t *resp; nxt_request_rpc_data_t *req_rpc_data; + /* + * An application response never legitimately carries a descriptor and + * this handler does not read msg->fd. The dispatcher closes what a + * handler leaves behind, so this is belt and braces; it is here because + * an application owns its end of the port socket, which makes this the + * one handler a hostile peer reaches on every single request. + */ + nxt_port_recv_msg_close_fds(msg); + req_rpc_data = data; r = req_rpc_data->request; diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index 28970a86f..b12e55f54 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -440,6 +440,9 @@ nxt_router_access_log_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, access_log->fd = msg->fd[0]; + /* The access log owns the descriptor now. */ + msg->fd[0] = -1; + nxt_work_queue_add(&task->thread->engine->fast_work_queue, nxt_router_conf_apply, task, tmcf, NULL); } @@ -610,6 +613,7 @@ nxt_router_access_log_reopen_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg, } nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; nxt_mp_release(reopen->mem_pool); } From aaf92f752ecb7007480fc21f948cba90f1824956 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 14:12:55 +0200 Subject: [PATCH 146/391] test: pin the descriptor leak in nxt_port_rpc_handler() Drives nxt_port_rpc_handler() against a port whose rpc_streams hash is empty, with real pipe descriptors attached to a stream nothing registered, and asserts they are closed -- fcntl(fd, F_GETFD) failing with EBADF -- rather than asserting anything about the handler's return. Covers both arms that reach the early return (nxt_lvlhsh_find and nxt_lvlhsh_delete), one and two descriptors, and a message carrying none. Fails without the fix with "find, one fd leaked fd[0]", and the test binary exits non-zero on failure, so the c-tests job catches a regression. nxt_router_response_ready_handler() is not covered: reaching it needs a full request and nxt_request_rpc_data_t fixture, well beyond what the neighbouring tests set up. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- auto/sources | 1 + src/test/nxt_port_rpc_fd_test.c | 212 ++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 4 files changed, 218 insertions(+) create mode 100644 src/test/nxt_port_rpc_fd_test.c diff --git a/auto/sources b/auto/sources index f851ed4d3..aebe79819 100644 --- a/auto/sources +++ b/auto/sources @@ -193,6 +193,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_ctrunc_test.c \ src/test/nxt_port_fd_test.c \ + src/test/nxt_port_rpc_fd_test.c \ " diff --git a/src/test/nxt_port_rpc_fd_test.c b/src/test/nxt_port_rpc_fd_test.c new file mode 100644 index 000000000..10644d9d3 --- /dev/null +++ b/src/test/nxt_port_rpc_fd_test.c @@ -0,0 +1,212 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership in nxt_port_rpc_handler() + * (src/nxt_port_rpc.c). + * + * A stream that has no registered handler falls through nxt_port_rpc_handler() + * without running one, and until the fix nothing closed the descriptors the + * message carried. The stream number is a value off the wire -- any peer can + * name a stream this process never registered -- so a compromised application + * could exhaust the router's descriptor table one such message at a time. + * + * nxt_port_read_msg_process() now also closes what a dispatch leaves behind, + * which covers this path a second time; the close this test pins keeps the + * reject path correct on its own terms. + * + * This lives beside nxt_port_fd_test.c rather than inside it: that test + * drives nxt_port_new_port_handler(), nxt_port_mmap_handler() and the + * dispatcher itself, all of which need a port hash and NEW_PORT-shaped + * message bodies. This one only needs a port with an empty rpc_streams + * hash and a stream number nothing has registered -- the "find" and + * "delete" arms of nxt_port_rpc_handler() both take that path with a + * plain zeroed message. + */ + +#include +#include +#include "nxt_tests.h" + +#include + + +static nxt_bool_t +nxt_port_rpc_fd_test_is_open(nxt_fd_t fd) +{ + return fcntl(fd, F_GETFD) != -1; +} + + +static void +nxt_port_rpc_fd_test_close(nxt_fd_t fd) +{ + if (fd != -1 && nxt_port_rpc_fd_test_is_open(fd)) { + (void) close(fd); + } +} + + +/* + * Runs nxt_port_rpc_handler() over an unregistered stream and checks that + * every real descriptor the message carried was closed, and that the slots + * were cleared the way a handler that consumes them would leave them. + * "last" selects between the two ways an unregistered stream is looked up: + * nxt_lvlhsh_delete() when the message is the last for its stream, + * nxt_lvlhsh_find() otherwise -- both reach the same early return. + */ +static nxt_int_t +nxt_port_rpc_fd_test_unregistered(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, uint32_t stream, uint8_t last, nxt_bool_t second_fd, + const char *name) +{ + int pipe_fds[2]; + nxt_fd_t fd0, fd1; + nxt_port_recv_msg_t msg; + + if (nxt_slow_path(pipe(pipe_fds) != 0)) { + nxt_log_alert(thr->log, "port rpc fd test failed to create a pipe"); + return NXT_ERROR; + } + + fd0 = pipe_fds[0]; + fd1 = second_fd ? pipe_fds[1] : -1; + + if (!second_fd) { + /* Not needed for this case; close the write end right away. */ + (void) close(pipe_fds[1]); + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.fd[0] = fd0; + msg.fd[1] = fd1; + msg.port_msg.stream = stream; + msg.port_msg.last = last; + msg.port_msg.type = _NXT_PORT_MSG_RPC_READY; + + nxt_port_rpc_handler(task, &msg); + + if (nxt_slow_path(nxt_port_rpc_fd_test_is_open(fd0))) { + nxt_log_alert(thr->log, "port rpc fd test: %s leaked fd[0]", name); + goto fail; + } + + if (second_fd && nxt_slow_path(nxt_port_rpc_fd_test_is_open(fd1))) { + nxt_log_alert(thr->log, "port rpc fd test: %s leaked fd[1]", name); + goto fail; + } + + if (nxt_slow_path(msg.fd[0] != -1 || msg.fd[1] != -1)) { + nxt_log_alert(thr->log, "port rpc fd test: %s left fds in the " + "message", name); + return NXT_ERROR; + } + + return NXT_OK; + +fail: + + /* Close only what is still open; a closed number may already be reused. */ + nxt_port_rpc_fd_test_close(fd0); + nxt_port_rpc_fd_test_close(fd1); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_port_rpc_fd_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_port_recv_msg_t msg; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port rpc fd test started"); + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + port = nxt_mp_zalloc(mp, sizeof(nxt_port_t)); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * port->rpc_streams starts zeroed, which is exactly "no stream + * registered" for nxt_lvlhsh: no insert has ever happened. + */ + port->mem_pool = mp; + + /* The stream number itself is arbitrary: nothing ever registers it. */ + ret = nxt_port_rpc_fd_test_unregistered(thr, task, port, 42, 0, 0, + "find, one fd"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_rpc_fd_test_unregistered(thr, task, port, 42, 0, 1, + "find, two fds"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_rpc_fd_test_unregistered(thr, task, port, 43, 1, 0, + "delete, one fd"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_rpc_fd_test_unregistered(thr, task, port, 43, 1, 1, + "delete, two fds"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + /* + * A message with no descriptor attached (fd[0] == -1, as every real + * receive path sets it when the peer sent none) must not be mistaken + * for one that needs closing. + */ + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.stream = 44; + msg.port_msg.last = 0; + msg.port_msg.type = _NXT_PORT_MSG_RPC_READY; + + nxt_port_rpc_handler(task, &msg); + + if (nxt_slow_path(msg.fd[0] != -1 || msg.fd[1] != -1)) { + nxt_log_alert(thr->log, "port rpc fd test: a message without " + "descriptors was mishandled"); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_OK; + +done: + + nxt_mp_destroy(mp); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port rpc fd test passed"); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 98f1cbd21..90f890ace 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -257,6 +257,10 @@ main(int argc, char **argv) return 1; } + if (nxt_port_rpc_fd_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CGROUP) if (nxt_cgroup_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 54a0326b7..ef52f1ef3 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -88,6 +88,7 @@ nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); +nxt_int_t nxt_port_rpc_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From eafe83d6a5f73f0622e334d53a3b9e3423ff4aa8 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 20:00:23 +0200 Subject: [PATCH 147/391] test: pin the dispatcher tail close and fragment-merge close nxt_port_read_msg_process() (src/nxt_port_socket.c) now closes whatever a dispatch leaves in msg->fd[], plus one close in the last-fragment merge before msg->fd[0] is overwritten by the assembled message's own descriptors. Neither was covered: the existing tests all drive a handler directly, never the dispatcher's own tail and fragment paths. Adds nxt_port_fd_test_tail_close() and nxt_port_fd_test_frag_merge() to src/test/nxt_port_fd_test.c, exposed via a new NXT_TESTS-only wrapper, nxt_port_test_run_read_msg_process() (src/nxt_port_socket.c, declared in src/nxt_port.h), following nxt_main_test_run_start_process_handler()'s shape exactly. Three cases, using two new test handlers (.oosm ignores the message outright, .status blanks fd[0] without closing it -- a "keeps nothing" and a "keeps fd[0]" handler): *) Tail close, handler ignores the message: both real descriptors end up closed and both msg->fd[] slots at -1. Reverting nxt_port_socket.c to 63a992f8^ (the fix's parent) and rebuilding reproduces the pre-fix behaviour; the test fails with "port fd test: dispatch to a handler that ignores the message left fd[0] open". *) Tail close, handler keeps fd[0]: fd[0] stays open, fd[1] (never taken) is closed. This already passes on the pre-fix revision -- it exists to catch a future unconditional close, not a current leak, and is documented as such in the test's comment rather than presented as catching the bug. *) Fragment merge: a first fragment (nf=0, mf=1) carries descriptor A, a last fragment (nf=1, mf=0) on the same (stream, pid) carries a different descriptor B. With the ignoring handler, both A and B end up closed (A via the tail close once fmsg->fd[0] is dispatched, B via the merge close before the overwrite). With the keeping handler, A stays open and B is closed. Deleting only the merge close (nxt_port_close_fds(msg->fd) before msg->fd[0] = fmsg->fd[0]) and rebuilding, with the ignoring handler, reproduces "port fd test: fragment merge, handler ignores the message leaked B, the last fragment's own descriptor" -- confirming that assertion is the one that actually depends on the merge close, not on the tail close it sits next to. Both revert experiments used a scratch copy of nxt_port_socket.c and were discarded before this commit; the working tree here has only the fix and the new test. ./build/tests exits non-zero on either injected regression. test/test_static.py -q still passes (23 passed, 1 skipped). nxt_router_response_ready_handler() stays uncovered, as before: it needs a full request and nxt_request_rpc_data_t fixture well beyond this file's scope. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- src/nxt_port.h | 2 + src/nxt_port_socket.c | 17 +++ src/test/nxt_port_fd_test.c | 258 ++++++++++++++++++++++++++++++++++++ 3 files changed, 277 insertions(+) diff --git a/src/nxt_port.h b/src/nxt_port.h index 5d5d1c530..13ee4dec3 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -455,6 +455,8 @@ nxt_port_msg_cancel_t nxt_port_socket_cancel(nxt_task_t *task, #if (NXT_TESTS) void nxt_port_test_msg_alloc_failures(nxt_uint_t failures); void nxt_port_test_run_error_handler(nxt_task_t *task, nxt_port_t *port); +void nxt_port_test_run_read_msg_process(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg); /* * Counts entries into nxt_port_send_new_port(). It is nxt_inline and diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 89bc3e970..a4de22caf 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -66,6 +66,23 @@ nxt_port_test_run_error_handler(nxt_task_t *task, nxt_port_t *port) nxt_port_error_handler(task, &port->socket, NULL); } + +/* + * Public wrapper that lets src/test/nxt_port_fd_test.c invoke the static + * nxt_port_read_msg_process() directly with a synthesised port and message + * -- used to verify the tail close of whatever a dispatch leaves in + * msg->fd[], and the close in the last-fragment merge that runs before + * msg->fd[0] is overwritten by the assembled message's own descriptors + * (issue freeunitorg/freeunit#... "port: close the descriptors a handler + * does not take"). + */ +void +nxt_port_test_run_read_msg_process(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg) +{ + nxt_port_read_msg_process(task, port, msg); +} + #endif diff --git a/src/test/nxt_port_fd_test.c b/src/test/nxt_port_fd_test.c index 5e93fc1e1..f7e0c31be 100644 --- a/src/test/nxt_port_fd_test.c +++ b/src/test/nxt_port_fd_test.c @@ -26,6 +26,15 @@ * The one path this cannot reach is nxt_runtime_process_port_create() * returning NULL, which needs an allocation failure the test harness has no * way to inject. + * + * nxt_port_fd_test_tail_close() and nxt_port_fd_test_frag_merge() go one + * level up, into the static nxt_port_read_msg_process() itself (via the + * NXT_TESTS wrapper nxt_port_test_run_read_msg_process()), for the two + * closes nothing above exercises: the tail close that reclaims whatever a + * dispatch left behind, and the close in the last-fragment merge that runs + * before msg->fd[0] is overwritten by the assembled message's own + * descriptors. nxt_router_response_ready_handler() stays uncovered here + * too, for the same reason as above. */ #include @@ -63,8 +72,41 @@ nxt_port_fd_test_dispatch_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * A handler that takes fd[0] -- blanking its slot without closing it, the + * way a handler that keeps a descriptor is required to -- and releases + * fd[1] as usual. Used to pin that nxt_port_read_msg_process() leaves a + * kept descriptor alone: the tail close and the fragment-merge close both + * have to treat "-1" as "already spoken for", not "nothing was here". + */ +static void +nxt_port_fd_test_keep_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + msg->fd[0] = -1; + + if (msg->fd[1] != -1) { + nxt_fd_close(msg->fd[1]); + msg->fd[1] = -1; + } +} + + +/* + * A handler that ignores the message entirely -- the shape GET_MMAP and + * OOSM have, and the shape every handler had before the fix. It never + * touches msg->fd[], so whatever nxt_port_read_msg_process() does with a + * dispatch's leftovers is the only thing standing between this and a leak. + */ +static void +nxt_port_fd_test_ignore_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ +} + + static const nxt_port_handlers_t nxt_port_fd_test_handlers = { .quit = nxt_port_fd_test_dispatch_handler, + .status = nxt_port_fd_test_keep_handler, + .oosm = nxt_port_fd_test_ignore_handler, }; @@ -139,6 +181,188 @@ nxt_port_fd_test_closed(nxt_thread_t *thr, nxt_task_t *task, } +/* + * Runs a single, non-fragmented message through nxt_port_read_msg_process() + * and checks what the dispatcher's tail close left behind: this pins the + * close at the tail of the function, which runs after the handler and + * covers whatever it did not take -- distinct from nxt_port_fd_test_closed() + * above, which only exercises the dispatcher's own NULL-slot guard. + * + * "keep_fd0" selects the handler dispatched to: nxt_port_fd_test_handlers + * .oosm ignores the message outright (keep_fd0 false), .status blanks fd[0] + * without closing it (keep_fd0 true). Either way fd[1] is never taken, so + * it is always expected closed. + */ +static nxt_int_t +nxt_port_fd_test_tail_close(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, nxt_bool_t keep_fd0, const char *name) +{ + int pipe0[2], pipe1[2]; + nxt_buf_t *b; + nxt_port_recv_msg_t msg; + + if (nxt_slow_path(pipe(pipe0) != 0 || pipe(pipe1) != 0)) { + nxt_log_alert(thr->log, "port fd test failed to create a pipe"); + return NXT_ERROR; + } + + /* Only the read end of each pipe travels in the message. */ + (void) close(pipe0[1]); + (void) close(pipe1[1]); + + b = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_buf_t)); + if (nxt_slow_path(b == NULL)) { + (void) close(pipe0[0]); + (void) close(pipe1[0]); + return NXT_ERROR; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t); + msg.fd[0] = pipe0[0]; + msg.fd[1] = pipe1[0]; + msg.port_msg.type = keep_fd0 ? _NXT_PORT_MSG_STATUS : _NXT_PORT_MSG_OOSM; + + nxt_port_test_run_read_msg_process(task, port, &msg); + + if (nxt_slow_path(keep_fd0 != nxt_port_fd_test_is_open(pipe0[0]))) { + nxt_log_alert(thr->log, "port fd test: %s left fd[0] %s", name, + keep_fd0 ? "closed" : "open"); + goto fail; + } + + if (nxt_slow_path(nxt_port_fd_test_is_open(pipe1[0]))) { + nxt_log_alert(thr->log, "port fd test: %s leaked fd[1]", name); + goto fail; + } + + if (nxt_slow_path(msg.fd[0] != -1 || msg.fd[1] != -1)) { + nxt_log_alert(thr->log, "port fd test: %s left fds in the message", + name); + goto fail; + } + + if (keep_fd0) { + /* Ours now: the dispatcher correctly left it alone. */ + (void) close(pipe0[0]); + } + + return NXT_OK; + +fail: + + nxt_port_fd_test_close(pipe0[0]); + nxt_port_fd_test_close(pipe1[0]); + + return NXT_ERROR; +} + + +/* + * Runs a first fragment carrying descriptor A and a last fragment carrying + * a different descriptor B through the same (stream, pid), and checks the + * close in the merge that runs just before msg->fd[0] is overwritten by the + * assembled message's own descriptors (fmsg->fd[0]). Without that close, + * B -- the last fragment's own descriptor, never seen by any handler -- is + * dropped on the floor when the overwrite happens. + * + * "keep_fd0" selects the same two handlers as nxt_port_fd_test_tail_close(): + * ignoring the message (A ends up closed by the tail close, since nothing + * took it) or blanking fd[0] (A is kept open). B is never handed to a + * handler at all -- it is always expected closed. + */ +static nxt_int_t +nxt_port_fd_test_frag_merge(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, nxt_bool_t keep_fd0, const char *name) +{ + int pipe_a[2], pipe_b[2]; + nxt_buf_t *b; + nxt_port_recv_msg_t msg; + + if (nxt_slow_path(pipe(pipe_a) != 0 || pipe(pipe_b) != 0)) { + nxt_log_alert(thr->log, "port fd test failed to create a pipe"); + return NXT_ERROR; + } + + (void) close(pipe_a[1]); + (void) close(pipe_b[1]); + + /* First fragment: nf=0, mf=1, carries A. */ + + b = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_buf_t)); + if (nxt_slow_path(b == NULL)) { + (void) close(pipe_a[0]); + (void) close(pipe_b[0]); + return NXT_ERROR; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t); + msg.fd[0] = pipe_a[0]; + msg.fd[1] = -1; + msg.port_msg.stream = 0x46524147; /* "FRAG" -- distinct from other tests */ + msg.port_msg.pid = nxt_pid; + msg.port_msg.mf = 1; + msg.port_msg.type = keep_fd0 ? _NXT_PORT_MSG_STATUS : _NXT_PORT_MSG_OOSM; + + nxt_port_test_run_read_msg_process(task, port, &msg); + + /* Last fragment: nf=1, mf=0, carries a *different* descriptor, B. */ + + b = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_buf_t)); + if (nxt_slow_path(b == NULL)) { + nxt_port_fd_test_close(pipe_a[0]); + (void) close(pipe_b[0]); + return NXT_ERROR; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t); + msg.fd[0] = pipe_b[0]; + msg.fd[1] = -1; + msg.port_msg.stream = 0x46524147; + msg.port_msg.pid = nxt_pid; + msg.port_msg.nf = 1; + + nxt_port_test_run_read_msg_process(task, port, &msg); + + if (nxt_slow_path(keep_fd0 != nxt_port_fd_test_is_open(pipe_a[0]))) { + nxt_log_alert(thr->log, "port fd test: %s left A %s", name, + keep_fd0 ? "closed" : "open"); + goto fail; + } + + if (nxt_slow_path(nxt_port_fd_test_is_open(pipe_b[0]))) { + nxt_log_alert(thr->log, "port fd test: %s leaked B, the last " + "fragment's own descriptor", name); + goto fail; + } + + if (keep_fd0) { + /* Ours now: the handler kept it, the merge did not touch it. */ + (void) close(pipe_a[0]); + } + + return NXT_OK; + +fail: + + nxt_port_fd_test_close(pipe_a[0]); + nxt_port_fd_test_close(pipe_b[0]); + + return NXT_ERROR; +} + + nxt_int_t nxt_port_fd_test(nxt_thread_t *thr) { @@ -437,6 +661,40 @@ nxt_port_fd_test(nxt_thread_t *thr) goto done; } + /* + * nxt_port_read_msg_process() itself: the tail close that reclaims + * whatever a dispatch left in msg->fd[], and the close in the + * last-fragment merge that runs before msg->fd[0] is overwritten by the + * assembled message's own descriptors. + */ + + ret = nxt_port_fd_test_tail_close(thr, task, port, 0, + "dispatch to a handler that ignores " + "the message"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_fd_test_tail_close(thr, task, port, 1, + "dispatch to a handler that keeps " + "fd[0]"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_fd_test_frag_merge(thr, task, port, 0, + "fragment merge, handler ignores the " + "message"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = nxt_port_fd_test_frag_merge(thr, task, port, 1, + "fragment merge, handler keeps fd[0]"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + ret = NXT_OK; done: From 6f3c01997eca34308ea650bd38eaa93333f25773 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 15:33:46 +0200 Subject: [PATCH 148/391] port: report an inline write that EAGAIN left nowhere to go With the port writable and its queue empty, nxt_port_msg_chk_insert() declines and nxt_port_socket_write2() sends inline. If the socket answers EAGAIN and nxt_port_msg_insert_tail() then cannot allocate, the message is neither sent nor queued -- and write2() still answered NXT_OK, because nxt_port_write_handler() returns void and nothing carried a status back. A caller that answers an RPC and then sends a fallback loses both. The reply reads NXT_OK, so it clears the state that would have made the fallback carry the answer; and the fallback dies with it, because the EAGAIN cleared the port's write_ready, so that second message is queued rather than sent and the error handler scheduled by the first drains it. The port stays open throughout -- nxt_port_error_handler() completes buffers and fixes the use count, and closes nothing -- so nothing else reports it either. nxt_proto_child_exited() is the case that found this: its RPC_ERROR and the REMOVE_PID behind it are exactly that pair. Reporting it as NXT_ERROR is only safe if nothing was consumed, because src/nxt_port.h gives the caller back fd, fd2 and b on any answer but NXT_OK. On this exit nothing was: nxt_socketpair_send() sent no bytes and no descriptor, and nxt_port_msg_close_fd() runs only after a successful send. So leave the message whole and answer NXT_ERROR, which is the state every caller already handles for the allocation failure in chk_insert(). Only while this is still the first fragment. Once one has gone out, ->nf is set, the descriptor is closed and the sent buffers completed, so the message is no longer the caller's to clean up -- a payload over max_size, or a chain longer than NXT_IOBUF_MAX, reaches that. There the port keeps it and nxt_port_msg_drop() finishes it, as before. Leaving the loop on that exit re-arms the write event, where the old "goto fail" raised the error handler. The socket is alive -- this was a full socket buffer and a failed allocation, not a dead peer -- and without the re-arm nothing sets write_ready again, so everything queued on that port from then on would wait for a writable event that never comes. A send that fails outright keeps its old treatment: the message is dropped and NXT_OK returned, even though that answer is wrong in the same way. That path is taken by every worker that dies with a send in flight, and the callers that ignore this return were written for it -- they hand over a buffer and rely on the port to complete it. Reporting it there would leak those buffers -- websocket frames among them are shared-memory chunks that no pool teardown reclaims -- so it belongs with those call sites rather than here. src/test/nxt_port_fail_test.c drives the real thing: a socketpair with a 4 KB send buffer filled to EAGAIN, then the allocation hook armed. It asserts the answer, that the descriptor is still open, that no completion was queued once the work queue has been drained, that the port's queue is empty, and that the write event was re-armed exactly once. A payload that reached the shared queue answers NXT_OK whatever becomes of the wake-up that follows it, because the ring has taken the buffer and queued its completion by then; nxt_runtime_port_send_quit() is a caller that would otherwise complete it twice. That case predates this change -- the allocation failure in nxt_port_msg_chk_insert() could already answer NXT_ERROR there -- and the lost wake-up with it: nxt_port_queue_send() raises "notify" only on the 0-to-1 transition, so the item waits for the next message that takes this port's socket path, which is when nxt_port_queue_read_handler() drains the ring. That wait is now alerted rather than silent. The re-arm itself can fail. Posted across engines it allocates the work item it carries, under the same pressure that sent the write down this path, and its result was discarded -- leaving write_ready at 0 with no event to set it again, which is the stall the re-arm exists to prevent. It now raises the error handler when the post fails, the way every other failure in this function does, and says so. Fixes: #335 Co-Authored-By: Claude Opus 5 --- CHANGES | 7 + docs/changes.xml | 11 ++ src/nxt_application.c | 26 ++- src/nxt_port_socket.c | 191 +++++++++++++++++--- src/nxt_router.c | 4 + src/test/nxt_port_fail_test.c | 326 ++++++++++++++++++++++++++++++++++ 6 files changed, 530 insertions(+), 35 deletions(-) diff --git a/CHANGES b/CHANGES index c87c94d42..ed1dff2f9 100644 --- a/CHANGES +++ b/CHANGES @@ -57,6 +57,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 prototype that was not coming, and the requests waiting on it were never failed. + *) Bugfix: the port layer reported a message as sent when it had been + neither sent nor queued. This happened when a socket answered + EAGAIN and no memory was left to hold the message for a later + attempt, so it takes memory pressure to reach. The sender then + cleared the state that would have retried or reported it, so a + request could wait for an answer that nothing would ever deliver. + *) Bugfix: an application worker that died before it finished starting was reported to nobody, so the start it was forked for stayed pending forever. Once the application reached its "processes" maximum it diff --git a/docs/changes.xml b/docs/changes.xml index eb49ac41f..09308b519 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -68,6 +68,17 @@ requests waiting on it were never failed. + + +the port layer reported a message as sent when it had been neither sent nor +queued. This happened when a socket answered EAGAIN and no memory was left to +hold the message for a later attempt, so it takes memory pressure to reach. +The sender then cleared the state that would have +retried or reported it, so a request could wait for an answer that nothing +would ever deliver. + + + an application worker that died before it finished starting was reported to diff --git a/src/nxt_application.c b/src/nxt_application.c index c94cec238..4f85da2dd 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -1364,20 +1364,18 @@ nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process) * the same pools and can fail the same way, and then only the * application's "limits.start_timeout" retires the start. The * alert below is what makes that case visible instead of silent. - * NXT_OK is not a delivery receipt either, and that shape is - * the one this function cannot cover. When the port is writable - * the message goes out inline; if sendmsg() answers EAGAIN and - * it then cannot be queued for later, it is dropped and only the - * port's error handler is scheduled -- while the caller reads - * NXT_OK. The REMOVE_PID below does not rescue that, and not - * because it is streamless: the EAGAIN cleared the port's - * write_ready, so it is queued rather than sent, and the error - * handler drains the whole queue when it runs. The router is - * told nothing at all. That is the port layer answering for a - * message it dropped, which reaches every RPC reply in the - * daemon rather than this function alone; it is filed as #335, - * and the fix there -- reporting the drop -- is what this - * function's failure branch above already knows how to use. + * One more shape reaches the same branch. The router port is + * writable, so the RPC_ERROR goes out inline and hits EAGAIN + * with no memory left to hold it for a later attempt. + * nxt_port_socket_write2() used to answer NXT_OK for that and + * drop the message, clearing the stream for a report nobody + * received. REMOVE_PID could not rescue it either: the same + * EAGAIN had cleared write_ready, so REMOVE_PID was queued + * instead of sent, and the error handler that ran next drained + * it too. It now answers NXT_ERROR with the message untouched, + * which keeps the stream armed here. A send to a port whose + * peer has died still answers NXT_OK; the RPC is moot there + * anyway. */ if (nxt_fast_path(ret == NXT_OK)) { process->stream = 0; diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index a4de22caf..9cdc13c06 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -20,6 +20,8 @@ static uint8_t nxt_port_enqueue_buf(nxt_task_t *task, nxt_port_msg_t *pm, static nxt_int_t nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, nxt_port_send_msg_t *msg); static nxt_port_send_msg_t *nxt_port_msg_alloc(const nxt_port_send_msg_t *m); +static nxt_int_t nxt_port_write_msgs(nxt_task_t *task, void *obj, + void *data); static void nxt_port_write_handler(nxt_task_t *task, void *obj, void *data); static nxt_port_send_msg_t *nxt_port_msg_first(nxt_port_t *port); nxt_inline void nxt_port_msg_close_fd(nxt_port_send_msg_t *msg); @@ -205,22 +207,22 @@ nxt_port_release_send_msg(nxt_port_send_msg_t *msg) /* - * Give up on a message that nxt_port_write_handler() is dropping, completing + * Give up on a message that nxt_port_write_msgs() is dropping. Complete * what it still owns. * - * Only for a message that is not in port->messages -- msg->link.next == NULL, - * which on these paths means it is still the caller's stack copy from the - * nxt_port_msg_chk_insert() NXT_DECLINED branch. That is also why this cannot - * double-complete: the nxt_port_error_handler() the caller goes on to raise - * completes what the queue holds, and this was never in it. + * This applies only to a message not yet in port->messages, msg->link.next + * == NULL: the caller's stack copy from the nxt_port_msg_chk_insert() + * NXT_DECLINED branch. It cannot double-complete, because the + * nxt_port_error_handler() the caller raises next completes only what the + * queue holds, and this message was never in it. * - * Without this the buffers of an accepted write were simply orphaned, which - * breaks the contract nxt_port_socket_write2() documents -- it answers NXT_OK - * on these paths, because nxt_port_write_handler() returns void -- and strands - * any caller waiting on the completion, such as the START_PROCESS deadline in - * src/nxt_router.c. + * nxt_port_socket_write2() answers NXT_OK on these paths on purpose. + * Callers that ignore the return rely on the port to finish what they + * handed it, such as the START_PROCESS deadline in src/nxt_router.c. + * Skipping this call would orphan those buffers instead. * - * Descriptors first, then buffers, the order nxt_port_error_handler() uses. + * Close descriptors first, then complete buffers -- the order + * nxt_port_error_handler() uses. */ static void @@ -258,12 +260,15 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, int notify; uint8_t qmsg_size; nxt_int_t res; + nxt_bool_t enqueued; nxt_port_send_msg_t msg; struct { nxt_port_msg_t pm; uint8_t buf[NXT_PORT_MAX_ENQUEUE_BUF_SIZE]; } qmsg; + enqueued = 0; + msg.link.next = NULL; msg.link.prev = NULL; @@ -315,6 +320,13 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, return res; } + /* + * The payload is in the shared queue and its completion is + * already queued: b belongs to the port from here on, whatever + * becomes of the wake-up below. + */ + enqueued = 1; + msg.port_msg.type = _NXT_PORT_MSG_READ_QUEUE; msg.buf = NULL; @@ -335,7 +347,48 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, res = nxt_port_msg_chk_insert(task, port, &msg); if (nxt_fast_path(res == NXT_DECLINED)) { - nxt_port_write_handler(task, &port->socket, &msg); + /* + * Inline, with this stack copy: the answer is whether the port took + * the message, not whether the call ran. On NXT_ERROR nothing of it + * was consumed, so the caller still owns fd, fd2 and b -- the + * contract in src/nxt_port.h, and the same state this function + * already answers for the allocation failure above. + */ + res = nxt_port_write_msgs(task, &port->socket, &msg); + } + + if (nxt_slow_path(res != NXT_OK && enqueued)) { + /* + * Only the wake-up failed. The buffer already went into the shared + * queue above, with its completion queued, so answering anything + * but NXT_OK would hand back a buffer this function no longer owns + * -- and a caller that cleans up after a failed write would + * complete it a second time. nxt_runtime_port_send_quit() + * (src/nxt_runtime.c) is such a caller. + * + * The message sits in the queue, but nothing told the peer to look. + * nxt_port_queue_send() raises "notify" only on the 0-to-1 + * transition (src/nxt_port_queue.h), so later enqueues on this port + * stay silent while this item keeps nitems above zero. + * + * After EAGAIN the peer still gets it. That errno means the socket + * holds data the peer has not read, and nxt_port_queue_read_handler() + * drains the ring before it reads the socket, so the next read pass + * takes this item too. + * + * ENOBUFS promises nothing: nxt_socketpair_send() maps it to + * NXT_AGAIN as well (src/nxt_socketpair.c), and it means the system + * ran out of buffers, not that the peer owes this port a read. The + * item then waits until something else makes that peer read. Keeping + * the notification instead needs one the port has already allocated, + * the way ->release_work is held for the release path; the alert + * marks the case until then. + */ + + nxt_alert(task, "port{%d,%d} %d: the queued message could not be " + "announced; the peer takes it only when it next reads this " + "port", (int) port->pid, (int) port->id, port->socket.fd); + res = NXT_OK; } @@ -464,12 +517,30 @@ nxt_port_fd_enable_write(nxt_task_t *task, nxt_port_t *port, void *data) } -static void -nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) +/* + * Write what the port has and report whether an inline message was taken. + * + * The event loop never sets the port socket's own ->data, so data != NULL + * marks the inline call from nxt_port_socket_write2(), with "msg" as that + * caller's stack copy. Only then does the return value matter -- a queued + * message belongs to the port either way, with nobody left to tell. + * + * NXT_ERROR means the message was neither sent nor queued, and nothing of + * it was consumed. See the ownership contract this promises the caller in + * the comment above the inline call in nxt_port_socket_write2(). + * + * One exit answers NXT_ERROR today: an EAGAIN with no memory to hold a + * first fragment for a later attempt. A send to a dead peer still answers + * NXT_OK and drops the message; see the comment on that path below. + */ + +static nxt_int_t +nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data) { int use_delta; size_t plain_size; ssize_t n; + nxt_int_t ret; uint32_t mmsg_buf[3 * NXT_IOBUF_MAX * 10]; nxt_bool_t block_write, enable_write; nxt_port_t *port; @@ -481,6 +552,7 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) port = nxt_container_of(obj, nxt_port_t, socket); + ret = NXT_OK; block_write = 0; enable_write = 0; use_delta = 0; @@ -620,11 +692,16 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) } else { /* - * Both failures below drop a message that was never queued, so - * both go through nxt_port_msg_drop(): the send is over and - * nothing else will ever complete its buffers. Neither is a - * socket-death-only path -- the second is an allocation failure - * against a port that is otherwise perfectly alive. + * A send that failed outright: the peer is gone. This keeps the + * old treatment -- drop the message here and answer NXT_OK -- + * even though the caller is then told a message went out that + * did not. It is the routine path, taken by every worker that + * dies with a send in flight, and the callers that ignore this + * return were written for it: they hand over a buffer and rely + * on the port to complete it. Reporting it instead would leak + * those buffers, one of them a chain of shared-memory chunks + * that no pool teardown reclaims. It has to be done together + * with those call sites; #348 tracks that. */ if (nxt_slow_path(n == NXT_ERROR)) { @@ -638,6 +715,38 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) if (msg->link.next == NULL) { qmsg = nxt_port_msg_insert_tail(port, msg); if (nxt_slow_path(qmsg == NULL)) { + /* + * EAGAIN, then no memory to hold the message for a later + * attempt. The socket is alive and nothing of the + * message was consumed: nxt_socketpair_send() sent no + * bytes and no descriptor, and nxt_port_msg_close_fd() + * runs only after a successful send. So an inline + * caller can be told, and can act on it: keep its stream + * armed, or complete its own payload. + * + * This applies only to the first fragment. Once a + * fragment has gone out, ->nf is set, the descriptor is + * closed, and the sent buffers are completed, so the + * port owns the cleanup, not the caller. + */ + + if (data != NULL && msg->port_msg.nf == 0) { + ret = NXT_ERROR; + + /* + * Leave the loop the ordinary way instead of raising + * the error handler. The socket is alive, and the + * ordinary exit re-arms the write event that + * nxt_socketpair_send() just cleared. Without that + * re-arm, everything queued on this port from here + * on -- including work another thread inserted + * during this send -- would wait for a writable + * event that never comes. + */ + + break; + } + nxt_port_msg_drop(task, msg); goto fail; @@ -671,12 +780,52 @@ nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) } if (enable_write) { - nxt_port_post(task, port, nxt_port_fd_enable_write, NULL); + /* + * Same engine, nxt_port_post() is a direct call and cannot fail. + * Cross-engine, it allocates the work item it posts, and that can + * fail under the same memory pressure that sent this write down the + * refusal path above. A re-arm that never happens leaves + * write_ready at 0 with no event to set it again, so anything on + * this port's queue -- inserted concurrently, or by the next caller + * -- would wait forever. + * + * Raise the error handler instead, as every other failure here + * does. It drains the queue and completes what it holds, so those + * callers get their buffers back. + * + * It does not repair the port. write_ready stays 0 with no write + * event armed, so every later message is queued and never sent until + * the port is closed. An embedded work item on the port, the way + * ->release_work already works, would let the re-arm run without an + * allocation and remove this branch. + */ + + if (nxt_slow_path(nxt_port_post(task, port, nxt_port_fd_enable_write, + NULL) != NXT_OK)) + { + nxt_alert(task, "port{%d,%d} %d: cannot re-arm the write event; " + "draining what it holds, and it sends nothing further", + (int) port->pid, (int) port->id, port->socket.fd); + + use_delta++; + + nxt_work_queue_add(wq, nxt_port_error_handler, task, + &port->socket, &port->socket); + } } if (use_delta != 0) { nxt_port_use(task, port, use_delta); } + + return ret; +} + + +static void +nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) +{ + (void) nxt_port_write_msgs(task, obj, data); } diff --git a/src/nxt_router.c b/src/nxt_router.c index 0bf43ff7b..888a564a5 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1017,6 +1017,10 @@ nxt_router_start_timeout(nxt_task_t *task, void *obj, void *data) * nxt_port_msg_drop() because the destination died or the port layer could not * requeue it -- and nothing was forked for a start that never arrived. * + * One shape does not reach here. An inline first fragment that hits EAGAIN + * and cannot be held for a later attempt is refused outright, with + * NXT_ERROR and the payload untouched, so the caller fails the start itself. + * * ->recalled cannot answer this on its own: a message the deadline finds * partly sent stays queued, and only its completion knows whether the rest of * it ever left. See nxt_router_app_port_error(). diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index ddd070468..a13188531 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "nxt_tests.h" #if (NXT_LINUX) @@ -15,6 +16,9 @@ static nxt_port_t *nxt_port_fail_test_port(nxt_task_t *task); static nxt_int_t nxt_port_fail_test_socket_write(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_inline_drop(nxt_thread_t *thr); +static void nxt_port_fail_test_enable_write(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_error_handler(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_mp_baseline(nxt_thread_t *thr); @@ -41,6 +45,7 @@ static nxt_int_t nxt_port_fail_test_fd_count(void); static nxt_uint_t nxt_port_fail_test_completions; +static nxt_uint_t nxt_port_fail_test_rearms; static nxt_uint_t nxt_port_fail_test_releases; static nxt_uint_t nxt_port_fail_test_signals; @@ -55,6 +60,10 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_inline_drop(thr) != NXT_OK) { + return NXT_ERROR; + } + if (nxt_port_fail_test_rpc_register(thr) != NXT_OK) { return NXT_ERROR; } @@ -229,6 +238,323 @@ nxt_port_fail_test_socket_write(nxt_thread_t *thr) } +static void +nxt_port_fail_test_enable_write(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_port_fail_test_rearms++; +} + + +/* + * The inline write that the port cannot take. + * + * The other leg above drives the allocation failure in + * nxt_port_msg_chk_insert(), before anything is sent. This one drives the + * other half: the port is write-ready with an empty queue, so chk_insert() + * declines and nxt_port_socket_write2() sends inline, and there the socket + * answers EAGAIN with the message unable to queue for later either. + * + * Nothing of the message was consumed on that path, so the answer must be + * NXT_ERROR, with the descriptor still open and no completion queued: the + * ownership contract in src/nxt_port.h, which the caller relies on to free + * its own payload. Answering NXT_OK there is what left an RPC reply lost, + * with the caller believing it had sent it (#335). + * + * The write event must also be re-armed. nxt_socketpair_send() cleared + * write_ready on the EAGAIN, and nothing else sets it back, so a port that + * skipped the re-arm would never drain what is already queued on it. The + * stub engine below exists to count that call. + */ + +static nxt_int_t +nxt_port_fail_test_inline_drop(nxt_thread_t *thr) +{ + int sndbuf; + nxt_mp_t *mp; + nxt_fd_t fd, pair[2]; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine, *saved_engine; + nxt_port_queue_t *queue; + nxt_event_interface_t stub; + u_char block[4096]; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + fd = -1; + pair[0] = -1; + pair[1] = -1; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* + * Only fast_work_queue and the one event method this path can reach are + * real. The code under test never reaches anything else, so it stays + * unset. + */ + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_fail_test_enable_write; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + /* + * port->engine matches the caller's, so nxt_port_post() is a direct + * call. The re-arm then lands on the stub above instead of on an event + * loop this fixture does not have. + */ + port->engine = &engine; + + /* + * SOCK_DGRAM: the type the port layer itself selects, since + * src/nxt_socketpair.c keeps SEQPACKET switched off. This fixture then + * builds a pair on any platform the product supports. It needs only a + * send buffer it can fill, not seqpacket semantics. + */ + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: socketpair failed"); + goto done; + } + + sndbuf = 4096; + (void) setsockopt(pair[1], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf)); + + if (nxt_slow_path(fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: O_NONBLOCK failed"); + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.fd = pair[1]; + port->socket.task = task; + port->socket.log = thr->log; + port->max_size = 1024; + port->max_share = 1024; + + /* Nobody reads pair[0], so the send buffer fills and stays full. */ + + nxt_memzero(block, sizeof(block)); + + while (send(pair[1], block, sizeof(block), 0) > 0) { + /* void */ + } + + port->socket.write_ready = 1; + port->socket.write = NXT_EVENT_INACTIVE; + + fd = open("/dev/null", O_RDONLY); + if (nxt_slow_path(fd == -1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: failed to open /dev/null"); + goto done; + } + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: failed to allocate buf"); + goto done; + } + + buf->completion_handler = nxt_port_fail_test_completion; + + nxt_port_fail_test_completions = 0; + nxt_port_fail_test_rearms = 0; + + nxt_port_test_msg_alloc_failures(1); + + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA + | NXT_PORT_MSG_CLOSE_FD, fd, 1, 0, buf); + + nxt_port_test_msg_alloc_failures(0); + + if (ret != NXT_ERROR) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: an inline write that was neither " + "sent nor queued answered %d, expected NXT_ERROR", + (int) ret); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_ERROR; + + if (!nxt_port_fail_test_fd_is_open(fd)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the inline drop closed a descriptor " + "it reported as not taken"); + fd = -1; + goto done; + } + + /* + * Drain first: both consuming paths queue the completion rather than run + * it, so a counter read before this would be 0 whether or not the port + * took the buffer -- an assertion that holds for the wrong reason. + */ + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_fail_test_completions != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the inline drop completed a buffer " + "the caller still owns"); + goto done; + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the refused message was left on the " + "port"); + goto done; + } + + if (port->socket.write_ready != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: write_ready survived an EAGAIN"); + goto done; + } + + if (nxt_port_fail_test_rearms != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the write event was re-armed %d " + "times, expected once -- without it nothing queued on " + "this port from here on would ever drain", + (int) nxt_port_fail_test_rearms); + goto done; + } + + /* + * The same failure, but on a port with a shared queue: the payload goes + * into the ring, and the inline write is only the wake-up that tells the + * reader to look. The buffer is already handed over, its completion + * already queued, so this one must answer NXT_OK however the wake-up + * ends. nxt_runtime_port_send_quit() is a caller that cleans up after a + * failed write; it would otherwise complete the same buffer twice. + */ + + queue = nxt_mp_zalloc(mp, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue == NULL)) { + ret = NXT_ERROR; + goto done; + } + + nxt_port_queue_init(queue); + port->queue = queue; + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + ret = NXT_ERROR; + goto done; + } + + buf->completion_handler = nxt_port_fail_test_completion; + buf->mem.free++; + + nxt_port_fail_test_completions = 0; + + /* As the poller would leave it; the socket buffer is still full. */ + port->socket.write_ready = 1; + + nxt_port_test_msg_alloc_failures(1); + + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 0, 0, buf); + + nxt_port_test_msg_alloc_failures(0); + + if (ret != NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: a payload that reached the shared " + "queue answered %d; the buffer is already the port's " + "and the caller must not be told to reclaim it", + (int) ret); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_ERROR; + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_fail_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the enqueued payload was completed " + "%d times, expected once", + (int) nxt_port_fail_test_completions); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_port_test_msg_alloc_failures(0); + + if (fd != -1 && nxt_port_fail_test_fd_is_open(fd)) { + nxt_fd_close(fd); + } + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + /* + * Close through the port's own teardown, not by closing the pair here. + * nxt_port_mp_cleanup() asserts both descriptors are gone, and that + * assertion compiles out of a release build, so closing them by hand + * looks clean until someone runs the suite with --debug. + */ + + /* + * Take the queue back first. It came from the test's own mem pool, and + * nxt_port_close() munmap()s whatever port->queue points at -- 655380 + * bytes of live heap if the allocator ever returns a page-aligned + * pointer. Today the call fails EINVAL and only logs, which is luck, + * not a design. + */ + + port->queue = NULL; + + nxt_port_close(task, port); + + if (pair[0] != -1 && nxt_port_fail_test_fd_is_open(pair[0])) { + nxt_fd_close(pair[0]); + } + + nxt_port_use(task, port, -1); + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_mp_destroy(mp); + + return ret; +} + + static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr) { From 47afaed6447277965f391f4afb4f11dae073aff8 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 22:28:36 +0200 Subject: [PATCH 149/391] test: one definition for "is this descriptor still open" Seven test files carried the same helper, each under its own prefix: static nxt_bool_t nxt__fd_is_open(nxt_fd_t fd) { return fcntl(fd, F_GETFD) != -1; } The prefixes exist because every test object links into one binary, not because the copies differ -- five were identical and the other two only renamed it. Two more files spelled the same test out by hand. Put it in src/test/nxt_tests.c with a prototype in nxt_tests.h, where the suite already keeps shared support, and drop the copies. A definition in a translation unit rather than a header also keeps the build quiet: a static helper in a header would want an inline marker and would trip -Wunused-function in every file that did not call it, and the tree builds tests with -Wall -Wextra -Werror. src/test/nxt_unit_port_recv_test.c keeps its own spelling. It is not part of the tests binary -- auto/make links it from its own object alone -- so it cannot reach nxt_tests.c. No behaviour changes: the helper is the same expression it replaced, and the suite runs 50/50 in release and debug. Co-Authored-By: Claude Opus 5 --- src/test/nxt_main_start_process_reply_test.c | 27 ++++++---------- src/test/nxt_port_change_file_test.c | 25 ++++++--------- src/test/nxt_port_ctrunc_test.c | 2 +- src/test/nxt_port_fail_test.c | 22 +++++-------- src/test/nxt_port_fd_test.c | 27 ++++++---------- src/test/nxt_port_ready_test.c | 33 ++++++++------------ src/test/nxt_port_rpc_fd_test.c | 13 ++------ src/test/nxt_router_new_port_test.c | 13 +------- src/test/nxt_tests.c | 13 ++++++++ src/test/nxt_tests.h | 2 ++ 10 files changed, 69 insertions(+), 108 deletions(-) diff --git a/src/test/nxt_main_start_process_reply_test.c b/src/test/nxt_main_start_process_reply_test.c index 7f610a7b0..29c9eabeb 100644 --- a/src/test/nxt_main_start_process_reply_test.c +++ b/src/test/nxt_main_start_process_reply_test.c @@ -64,13 +64,6 @@ #include -static nxt_bool_t -nxt_main_start_process_reply_test_fd_is_open(nxt_fd_t fd) -{ - return fcntl(fd, F_GETFD) != -1; -} - - /* * Run the handler once and require that it answered: exactly one RPC_ERROR * on reply_port, carrying the stream of the incoming message, and nothing @@ -113,12 +106,12 @@ nxt_main_start_process_reply_test_case(nxt_thread_t *thr, nxt_task_t *task, * the fix, only a guard against the cleanup below closing a number that * has already been reused. */ - if (msg->fd[0] == -1 && !nxt_main_start_process_reply_test_fd_is_open(fd0)) + if (msg->fd[0] == -1 && !nxt_test_fd_is_open(fd0)) { fd0 = -1; } - if (msg->fd[1] == -1 && !nxt_main_start_process_reply_test_fd_is_open(fd1)) + if (msg->fd[1] == -1 && !nxt_test_fd_is_open(fd1)) { fd1 = -1; } @@ -180,11 +173,11 @@ nxt_main_start_process_reply_test_case(nxt_thread_t *thr, nxt_task_t *task, * Close only what is still open: a consumed descriptor is gone already, * and closing its number again could reach an unrelated file. */ - if (fd0 != -1 && nxt_main_start_process_reply_test_fd_is_open(fd0)) { + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { (void) close(fd0); } - if (fd1 != -1 && nxt_main_start_process_reply_test_fd_is_open(fd1)) { + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { (void) close(fd1); } @@ -235,12 +228,12 @@ nxt_main_start_process_reply_test_no_reply_case(nxt_thread_t *thr, nxt_main_test_run_start_process_handler(task, msg); - if (msg->fd[0] == -1 && !nxt_main_start_process_reply_test_fd_is_open(fd0)) + if (msg->fd[0] == -1 && !nxt_test_fd_is_open(fd0)) { fd0 = -1; } - if (msg->fd[1] == -1 && !nxt_main_start_process_reply_test_fd_is_open(fd1)) + if (msg->fd[1] == -1 && !nxt_test_fd_is_open(fd1)) { fd1 = -1; } @@ -264,11 +257,11 @@ nxt_main_start_process_reply_test_no_reply_case(nxt_thread_t *thr, done: - if (fd0 != -1 && nxt_main_start_process_reply_test_fd_is_open(fd0)) { + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { (void) close(fd0); } - if (fd1 != -1 && nxt_main_start_process_reply_test_fd_is_open(fd1)) { + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { (void) close(fd1); } @@ -357,8 +350,8 @@ nxt_main_start_process_reply_test_proto_case(nxt_thread_t *thr, } if (msg->fd[0] != -1 || msg->fd[1] != -1 - || nxt_main_start_process_reply_test_fd_is_open(fd0) - || nxt_main_start_process_reply_test_fd_is_open(fd1)) + || nxt_test_fd_is_open(fd0) + || nxt_test_fd_is_open(fd1)) { nxt_log_alert(thr->log, "main start process reply test: %s leaked " "a descriptor", name); diff --git a/src/test/nxt_port_change_file_test.c b/src/test/nxt_port_change_file_test.c index 10b67b292..1b5e1b05d 100644 --- a/src/test/nxt_port_change_file_test.c +++ b/src/test/nxt_port_change_file_test.c @@ -31,13 +31,6 @@ #include -static nxt_bool_t -nxt_port_change_file_test_fd_is_open(nxt_fd_t fd) -{ - return fcntl(fd, F_GETFD) != -1; -} - - /* * Feed the handler a message it has to refuse, and check that nothing the * message carried outlives the call. @@ -67,8 +60,8 @@ nxt_port_change_file_test_reject(nxt_thread_t *thr, nxt_task_t *task, nxt_port_change_log_file_handler(task, msg); - if ((fd0 != -1 && nxt_port_change_file_test_fd_is_open(fd0)) - || nxt_port_change_file_test_fd_is_open(fd1)) + if ((fd0 != -1 && nxt_test_fd_is_open(fd0)) + || nxt_test_fd_is_open(fd1)) { nxt_log_alert(thr->log, "port change file test: %s leaked a " "descriptor", name); @@ -90,11 +83,11 @@ nxt_port_change_file_test_reject(nxt_thread_t *thr, nxt_task_t *task, * are gone already, and closing them again could reach an unrelated * descriptor that reused the number. */ - if (fd0 != -1 && nxt_port_change_file_test_fd_is_open(fd0)) { + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { (void) close(fd0); } - if (fd1 != -1 && nxt_port_change_file_test_fd_is_open(fd1)) { + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { (void) close(fd1); } @@ -128,8 +121,8 @@ nxt_port_change_file_test_accept(nxt_thread_t *thr, nxt_task_t *task, nxt_port_change_log_file_handler(task, msg); - if (nxt_port_change_file_test_fd_is_open(fd0) - || nxt_port_change_file_test_fd_is_open(fd1)) + if (nxt_test_fd_is_open(fd0) + || nxt_test_fd_is_open(fd1)) { nxt_log_alert(thr->log, "port change file test: %s leaked a " "descriptor", name); @@ -142,7 +135,7 @@ nxt_port_change_file_test_accept(nxt_thread_t *thr, nxt_task_t *task, return NXT_ERROR; } - if (!nxt_port_change_file_test_fd_is_open(log_file->fd)) { + if (!nxt_test_fd_is_open(log_file->fd)) { nxt_log_alert(thr->log, "port change file test: %s closed the log " "file", name); return NXT_ERROR; @@ -152,11 +145,11 @@ nxt_port_change_file_test_accept(nxt_thread_t *thr, nxt_task_t *task, fail: - if (fd0 != -1 && nxt_port_change_file_test_fd_is_open(fd0)) { + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { (void) close(fd0); } - if (fd1 != -1 && nxt_port_change_file_test_fd_is_open(fd1)) { + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { (void) close(fd1); } diff --git a/src/test/nxt_port_ctrunc_test.c b/src/test/nxt_port_ctrunc_test.c index ec7da95a9..3e489d843 100644 --- a/src/test/nxt_port_ctrunc_test.c +++ b/src/test/nxt_port_ctrunc_test.c @@ -60,7 +60,7 @@ nxt_port_ctrunc_test_fd_count(void) n = 0; for (fd = 0; fd < NXT_CTRUNC_TEST_FD_MAX; fd++) { - if (fcntl(fd, F_GETFD) != -1) { + if (nxt_test_fd_is_open(fd)) { n++; } } diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index a13188531..b5e893f6d 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -40,7 +40,6 @@ static void nxt_port_fail_test_mp_completion(nxt_task_t *task, void *obj, static void nxt_port_fail_test_completion(nxt_task_t *task, void *obj, void *data); static void nxt_port_fail_test_drain_wq(nxt_work_queue_t *wq); -static nxt_bool_t nxt_port_fail_test_fd_is_open(nxt_fd_t fd); static nxt_int_t nxt_port_fail_test_fd_count(void); @@ -185,7 +184,7 @@ nxt_port_fail_test_socket_write(nxt_thread_t *thr) nxt_port_test_msg_alloc_failures(0); - if (!nxt_port_fail_test_fd_is_open(fd)) { + if (!nxt_test_fd_is_open(fd)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, "port failure test closed fd before ownership transfer"); goto fail_close_port; @@ -221,7 +220,7 @@ nxt_port_fail_test_socket_write(nxt_thread_t *thr) fail_close_fd: - if (fd != -1 && nxt_port_fail_test_fd_is_open(fd)) { + if (fd != -1 && nxt_test_fd_is_open(fd)) { nxt_fd_close(fd); } @@ -404,7 +403,7 @@ nxt_port_fail_test_inline_drop(nxt_thread_t *thr) ret = NXT_ERROR; - if (!nxt_port_fail_test_fd_is_open(fd)) { + if (!nxt_test_fd_is_open(fd)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, "port failure test: the inline drop closed a descriptor " "it reported as not taken"); @@ -515,7 +514,7 @@ nxt_port_fail_test_inline_drop(nxt_thread_t *thr) nxt_port_test_msg_alloc_failures(0); - if (fd != -1 && nxt_port_fail_test_fd_is_open(fd)) { + if (fd != -1 && nxt_test_fd_is_open(fd)) { nxt_fd_close(fd); } @@ -540,7 +539,7 @@ nxt_port_fail_test_inline_drop(nxt_thread_t *thr) nxt_port_close(task, port); - if (pair[0] != -1 && nxt_port_fail_test_fd_is_open(pair[0])) { + if (pair[0] != -1 && nxt_test_fd_is_open(pair[0])) { nxt_fd_close(pair[0]); } @@ -750,7 +749,7 @@ nxt_port_fail_test_error_handler(nxt_thread_t *thr) goto fail_port; } - if (nxt_port_fail_test_fd_is_open(fd)) { + if (nxt_test_fd_is_open(fd)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, "port failure test did not close the queued fd"); goto fail_port; @@ -789,7 +788,7 @@ nxt_port_fail_test_error_handler(nxt_thread_t *thr) nxt_queue_remove(&msg->link); nxt_free(msg); } - if (fd != -1 && nxt_port_fail_test_fd_is_open(fd)) { + if (fd != -1 && nxt_test_fd_is_open(fd)) { nxt_fd_close(fd); } nxt_port_use(task, port, -1); @@ -1606,13 +1605,6 @@ nxt_port_fail_test_drain_wq(nxt_work_queue_t *wq) } -static nxt_bool_t -nxt_port_fail_test_fd_is_open(nxt_fd_t fd) -{ - return fcntl(fd, F_GETFD) != -1; -} - - static nxt_int_t nxt_port_fail_test_fd_count(void) { diff --git a/src/test/nxt_port_fd_test.c b/src/test/nxt_port_fd_test.c index f7e0c31be..a16caa14a 100644 --- a/src/test/nxt_port_fd_test.c +++ b/src/test/nxt_port_fd_test.c @@ -110,17 +110,10 @@ static const nxt_port_handlers_t nxt_port_fd_test_handlers = { }; -static nxt_bool_t -nxt_port_fd_test_is_open(nxt_fd_t fd) -{ - return fcntl(fd, F_GETFD) != -1; -} - - static void nxt_port_fd_test_close(nxt_fd_t fd) { - if (fd != -1 && nxt_port_fd_test_is_open(fd)) { + if (fd != -1 && nxt_test_fd_is_open(fd)) { (void) close(fd); } } @@ -152,8 +145,8 @@ nxt_port_fd_test_closed(nxt_thread_t *thr, nxt_task_t *task, handler(task, msg); - if ((first_fd && nxt_port_fd_test_is_open(fd0)) - || nxt_port_fd_test_is_open(fd1)) + if ((first_fd && nxt_test_fd_is_open(fd0)) + || nxt_test_fd_is_open(fd1)) { nxt_log_alert(thr->log, "port fd test: %s leaked a descriptor", name); goto fail; /* closes only what is still open */ @@ -228,13 +221,13 @@ nxt_port_fd_test_tail_close(nxt_thread_t *thr, nxt_task_t *task, nxt_port_test_run_read_msg_process(task, port, &msg); - if (nxt_slow_path(keep_fd0 != nxt_port_fd_test_is_open(pipe0[0]))) { + if (nxt_slow_path(keep_fd0 != nxt_test_fd_is_open(pipe0[0]))) { nxt_log_alert(thr->log, "port fd test: %s left fd[0] %s", name, keep_fd0 ? "closed" : "open"); goto fail; } - if (nxt_slow_path(nxt_port_fd_test_is_open(pipe1[0]))) { + if (nxt_slow_path(nxt_test_fd_is_open(pipe1[0]))) { nxt_log_alert(thr->log, "port fd test: %s leaked fd[1]", name); goto fail; } @@ -335,13 +328,13 @@ nxt_port_fd_test_frag_merge(nxt_thread_t *thr, nxt_task_t *task, nxt_port_test_run_read_msg_process(task, port, &msg); - if (nxt_slow_path(keep_fd0 != nxt_port_fd_test_is_open(pipe_a[0]))) { + if (nxt_slow_path(keep_fd0 != nxt_test_fd_is_open(pipe_a[0]))) { nxt_log_alert(thr->log, "port fd test: %s left A %s", name, keep_fd0 ? "closed" : "open"); goto fail; } - if (nxt_slow_path(nxt_port_fd_test_is_open(pipe_b[0]))) { + if (nxt_slow_path(nxt_test_fd_is_open(pipe_b[0]))) { nxt_log_alert(thr->log, "port fd test: %s leaked B, the last " "fragment's own descriptor", name); goto fail; @@ -501,7 +494,7 @@ nxt_port_fd_test(nxt_thread_t *thr) goto done; } - if (nxt_slow_path(msg.fd[0] != -1 || nxt_port_fd_test_is_open(fd0))) { + if (nxt_slow_path(msg.fd[0] != -1 || nxt_test_fd_is_open(fd0))) { nxt_log_alert(thr->log, "port fd test: an existing port did not " "refuse the socket"); nxt_port_fd_test_close(fd1); @@ -509,7 +502,7 @@ nxt_port_fd_test(nxt_thread_t *thr) goto done; } - if (nxt_slow_path(msg.fd[1] != fd1 || !nxt_port_fd_test_is_open(fd1))) { + if (nxt_slow_path(msg.fd[1] != fd1 || !nxt_test_fd_is_open(fd1))) { nxt_log_alert(thr->log, "port fd test: the queue descriptor of a " "queueless existing port was not left to the caller"); ret = NXT_ERROR; @@ -594,7 +587,7 @@ nxt_port_fd_test(nxt_thread_t *thr) goto done; } - if (nxt_slow_path(msg.fd[1] != fd1 || !nxt_port_fd_test_is_open(fd1))) { + if (nxt_slow_path(msg.fd[1] != fd1 || !nxt_test_fd_is_open(fd1))) { nxt_log_alert(thr->log, "port fd test: the queue descriptor was not " "left to the caller"); ret = NXT_ERROR; diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index b86bc10e6..2260f5908 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -51,13 +51,6 @@ #endif -static nxt_bool_t -nxt_port_ready_test_fd_is_open(nxt_fd_t fd) -{ - return fcntl(fd, F_GETFD) != -1; -} - - static nxt_int_t nxt_port_ready_test_case(nxt_thread_t *thr, nxt_task_t *task, nxt_port_recv_msg_t *msg, const char *name) @@ -77,8 +70,8 @@ nxt_port_ready_test_case(nxt_thread_t *thr, nxt_task_t *task, nxt_port_process_ready_handler(task, msg); - if (nxt_port_ready_test_fd_is_open(fd0) - || nxt_port_ready_test_fd_is_open(fd1)) + if (nxt_test_fd_is_open(fd0) + || nxt_test_fd_is_open(fd1)) { nxt_log_alert(thr->log, "port ready test: %s leaked a descriptor", name); @@ -100,11 +93,11 @@ nxt_port_ready_test_case(nxt_thread_t *thr, nxt_task_t *task, * are gone already, and closing them again could reach an unrelated * descriptor that reused the number. */ - if (fd0 != -1 && nxt_port_ready_test_fd_is_open(fd0)) { + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { (void) close(fd0); } - if (fd1 != -1 && nxt_port_ready_test_fd_is_open(fd1)) { + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { (void) close(fd1); } @@ -192,8 +185,8 @@ nxt_port_ready_test_reject(nxt_thread_t *thr, nxt_task_t *task, return NXT_ERROR; } - if (nxt_port_ready_test_fd_is_open(fd0) - || nxt_port_ready_test_fd_is_open(fd1)) + if (nxt_test_fd_is_open(fd0) + || nxt_test_fd_is_open(fd1)) { nxt_log_alert(thr->log, "port ready test: %s leaked a descriptor", name); @@ -204,11 +197,11 @@ nxt_port_ready_test_reject(nxt_thread_t *thr, nxt_task_t *task, fail: - if (fd0 != -1 && nxt_port_ready_test_fd_is_open(fd0)) { + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { (void) close(fd0); } - if (fd1 != -1 && nxt_port_ready_test_fd_is_open(fd1)) { + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { (void) close(fd1); } @@ -355,13 +348,13 @@ nxt_port_ready_test_queue(nxt_thread_t *thr, nxt_task_t *task, return NXT_ERROR; } - if (nxt_port_ready_test_fd_is_open(spare)) { + if (nxt_test_fd_is_open(spare)) { nxt_log_alert(thr->log, "port ready test: %s leaked the spare " "descriptor", name); return NXT_ERROR; } - if (prev_queue != NULL && nxt_port_ready_test_fd_is_open(prev_fd)) { + if (prev_queue != NULL && nxt_test_fd_is_open(prev_fd)) { nxt_log_alert(thr->log, "port ready test: %s did not release the " "previous queue", name); return NXT_ERROR; @@ -442,7 +435,7 @@ nxt_port_ready_test_short_queue(nxt_thread_t *thr, nxt_task_t *task, return NXT_ERROR; } - if (msg->fd[0] != -1 || nxt_port_ready_test_fd_is_open(fd)) { + if (msg->fd[0] != -1 || nxt_test_fd_is_open(fd)) { nxt_log_alert(thr->log, "port ready test: short queue leaked its " "descriptor"); return NXT_ERROR; @@ -658,7 +651,7 @@ nxt_port_ready_test_queueless(nxt_thread_t *thr, nxt_task_t *task, nxt_port_process_ready_handler(task, msg); - if (msg->fd[0] != -1 || nxt_port_ready_test_fd_is_open(fd)) { + if (msg->fd[0] != -1 || nxt_test_fd_is_open(fd)) { nxt_log_alert(thr->log, "port ready test: the refused queue leaked " "its descriptor"); goto fail; @@ -786,7 +779,7 @@ nxt_port_ready_test_queueless(nxt_thread_t *thr, nxt_task_t *task, nxt_port_process_ready_handler(task, msg); - if (msg->fd[0] != -1 || nxt_port_ready_test_fd_is_open(fd)) { + if (msg->fd[0] != -1 || nxt_test_fd_is_open(fd)) { nxt_log_alert(thr->log, "port ready test: the retransmitted READY " "leaked its descriptor"); goto fail; diff --git a/src/test/nxt_port_rpc_fd_test.c b/src/test/nxt_port_rpc_fd_test.c index 10644d9d3..922ee2ed2 100644 --- a/src/test/nxt_port_rpc_fd_test.c +++ b/src/test/nxt_port_rpc_fd_test.c @@ -32,17 +32,10 @@ #include -static nxt_bool_t -nxt_port_rpc_fd_test_is_open(nxt_fd_t fd) -{ - return fcntl(fd, F_GETFD) != -1; -} - - static void nxt_port_rpc_fd_test_close(nxt_fd_t fd) { - if (fd != -1 && nxt_port_rpc_fd_test_is_open(fd)) { + if (fd != -1 && nxt_test_fd_is_open(fd)) { (void) close(fd); } } @@ -89,12 +82,12 @@ nxt_port_rpc_fd_test_unregistered(nxt_thread_t *thr, nxt_task_t *task, nxt_port_rpc_handler(task, &msg); - if (nxt_slow_path(nxt_port_rpc_fd_test_is_open(fd0))) { + if (nxt_slow_path(nxt_test_fd_is_open(fd0))) { nxt_log_alert(thr->log, "port rpc fd test: %s leaked fd[0]", name); goto fail; } - if (second_fd && nxt_slow_path(nxt_port_rpc_fd_test_is_open(fd1))) { + if (second_fd && nxt_slow_path(nxt_test_fd_is_open(fd1))) { nxt_log_alert(thr->log, "port rpc fd test: %s leaked fd[1]", name); goto fail; } diff --git a/src/test/nxt_router_new_port_test.c b/src/test/nxt_router_new_port_test.c index 95dc095b1..e26a62fb9 100644 --- a/src/test/nxt_router_new_port_test.c +++ b/src/test/nxt_router_new_port_test.c @@ -48,17 +48,6 @@ #endif -#if (NXT_HAVE_MEMFD_CREATE) - -static nxt_bool_t -nxt_router_new_port_test_fd_is_open(nxt_fd_t fd) -{ - return fcntl(fd, F_GETFD) != -1; -} - -#endif - - nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr) { @@ -209,7 +198,7 @@ nxt_router_new_port_test(nxt_thread_t *thr) nxt_router_new_port_handler(task, &msg); - if (msg.fd[1] != -1 || nxt_router_new_port_test_fd_is_open(queue)) { + if (msg.fd[1] != -1 || nxt_test_fd_is_open(queue)) { nxt_log_alert(thr->log, "router new port test: a refused queue " "leaked its descriptor"); goto done; diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 90f890ace..55d1d5de0 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -14,6 +14,19 @@ nxt_module_init_t nxt_init_modules[1]; nxt_uint_t nxt_init_modules_n; +/* + * Whether the descriptor is still open in this process. Several port tests + * assert on a descriptor's real state rather than on what a handler returned, + * because the bug they guard against is a close that did or did not happen. + */ + +nxt_bool_t +nxt_test_fd_is_open(nxt_fd_t fd) +{ + return fcntl(fd, F_GETFD) != -1; +} + + /* The function is defined here to prevent inline optimizations. */ static nxt_bool_t nxt_msec_less(nxt_msec_t first, nxt_msec_t second) diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index ef52f1ef3..d2a381163 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -92,5 +92,7 @@ nxt_int_t nxt_port_rpc_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); +nxt_bool_t nxt_test_fd_is_open(nxt_fd_t fd); + #endif /* _NXT_TESTS_H_INCLUDED_ */ From f288ee895e5e6666ed6c5b40bcd2993eb2710606 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 02:31:13 +0200 Subject: [PATCH 150/391] static: answer conditional requests with 304 Unit emits ETag and Last-Modified on every static response and then ignores every validator a client sends back: a matching If-None-Match still gets the full 200. Every repeat view of a page re-ships every byte of every image, which for a media-heavy site -- a WordPress uploads tree, say -- is most of the traffic. Implement the precondition ladder of RFC 9110 Sect. 13.2.2 for GET and HEAD, in the order that section mandates: If-Match, then If-Unmodified-Since, then If-None-Match, then If-Modified-Since. The later member of each pair is consulted only when the earlier one is absent, so a client that sends both a tag and a date is judged by the tag even when the tag does not match. A failed If-Match or If-Unmodified-Since is a 412; a matching If-None-Match or an unchanged If-Modified-Since is a 304. Entity-tag comparison follows Sect. 8.8.3.2: If-Match compares strongly, so a W/-prefixed tag never matches there, while If-None-Match compares weakly and strips the prefix. "*" matches any representation, but only as the whole field value -- the grammar is "*" / #entity-tag, not a list member. An unparsable date is ignored rather than treated as an error. The response framing needed no work: nxt_http_request_is_bodyless_final() already treats 304 as bodyless and the h1 sender already skips chunked framing and keeps the connection alive for it, so a 304 goes out with the validators, no Content-Length and no Transfer-Encoding. The 412 status line was already in the h1 table; only the enum constant was missing. One trap worth recording: request headers land in r->inline_fields and only spill into the r->fields list, so the obvious nxt_list_each(f, r->fields) walk compiles, runs, and silently matches nothing on an ordinary request. nxt_http_fields_first()/_next() cover both, as nxt_router.c does. test_static_conditional_ignored pinned the old behaviour and is replaced by four tests of the new one. test_static_range_ignored stays: Range is still unimplemented, and ignoring it is still correct. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- CHANGES | 11 ++ src/nxt_http.h | 1 + src/nxt_http_static.c | 256 +++++++++++++++++++++++++++++++++++++++++- test/test_static.py | 125 +++++++++++++++++---- 4 files changed, 368 insertions(+), 25 deletions(-) diff --git a/CHANGES b/CHANGES index ed1dff2f9..ce5a1043a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,17 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Feature: conditional requests for static files. A "share" action now + answers a GET or HEAD carrying "If-None-Match" or "If-Modified-Since" + with "304 Not Modified" when the file's validators still match, so a + client that already holds the file is no longer sent it again. Unit + emitted "ETag" and "Last-Modified" before, but ignored both when they + came back. Entity-tag comparison is weak and "*" matches any + representation; a present "If-None-Match" takes precedence over + "If-Modified-Since", per RFC 9110. "If-Match" and + "If-Unmodified-Since" are evaluated first and answer "412 Precondition + Failed" when they do not hold. + *) Security: a file descriptor sent to a port is no longer leaked when the message is dispatched and nothing takes ownership of it. An application holds the write end of the router's main port and the diff --git a/src/nxt_http.h b/src/nxt_http.h index 5571fd31c..d473a4c81 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -36,6 +36,7 @@ typedef enum { NXT_HTTP_NOT_ACCEPTABLE = 406, NXT_HTTP_REQUEST_TIMEOUT = 408, NXT_HTTP_LENGTH_REQUIRED = 411, + NXT_HTTP_PRECONDITION_FAILED = 412, NXT_HTTP_PAYLOAD_TOO_LARGE = 413, NXT_HTTP_URI_TOO_LONG = 414, NXT_HTTP_UPGRADE_REQUIRED = 426, diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 044464e55..0f0848230 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -167,6 +167,10 @@ static void nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx); static void nxt_http_static_next(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx, nxt_http_status_t status); +static nxt_http_status_t nxt_http_static_preconditions(nxt_http_request_t *r, + nxt_str_t *etag, nxt_time_t mtime); +static nxt_bool_t nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, + nxt_bool_t strong); #if (NXT_HAVE_OPENAT2) static u_char *nxt_http_static_chroot_match(u_char *chr, u_char *shr); #endif @@ -485,12 +489,12 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, struct tm tm; nxt_buf_t *fb; nxt_int_t ret; - nxt_str_t *shr, *index, exten, *mtype; + nxt_str_t *shr, *index, exten, *mtype, etag; nxt_uint_t level; nxt_file_t *f, file; nxt_file_info_t fi; nxt_http_field_t *field; - nxt_http_status_t status; + nxt_http_status_t status, pcond; nxt_router_conf_t *rtcf; nxt_http_action_t *action; nxt_work_handler_t body_handler; @@ -732,6 +736,35 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_file_size(&fi)) - p; + etag.start = field->value; + etag.length = field->value_length; + + pcond = nxt_http_static_preconditions(r, &etag, nxt_file_mtime(&fi)); + + if (pcond != NXT_HTTP_OK) { + nxt_file_close(task, f); + f = NULL; + + if (pcond == NXT_HTTP_PRECONDITION_FAILED) { + nxt_http_request_error(task, r, pcond); + return; + } + + /* + * A 304 carries the validators and nothing else. + * content_length_n is reset so no Content-Length is emitted, and + * no body handler is scheduled; the h1 framing already + * special-cases 304, so keep-alive survives and the response is + * never chunked (src/nxt_h1proto.c). + */ + r->status = NXT_HTTP_NOT_MODIFIED; + r->resp.content_length_n = -1; + + body_handler = NULL; + + goto send; + } + if (exten.start == NULL) { nxt_http_static_extract_extension(shr, &exten); } @@ -878,6 +911,8 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, body_handler = NULL; } +send: + nxt_http_request_header_send(task, r, body_handler, NULL); r->state = &nxt_http_static_send_state; @@ -893,6 +928,223 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, } +/* + * Conditional requests, RFC 9110 Sect. 13.2.2, evaluated in the order that + * section mandates: If-Match, then If-Unmodified-Since, then If-None-Match, + * then If-Modified-Since. Returns NXT_HTTP_OK to serve the file normally, + * NXT_HTTP_NOT_MODIFIED for a 304, or NXT_HTTP_PRECONDITION_FAILED for a 412. + * + * The later step in each pair is consulted only when the earlier one is + * absent: a client that sends both an entity-tag and a date is asking to be + * judged by the entity-tag, even when the tag does not match. + */ + +static nxt_http_status_t +nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, + nxt_time_t mtime) +{ + nxt_str_t value; + nxt_time_t date; + nxt_http_field_t *f, *im, *ium, *inm, *ims; + nxt_http_fields_iter_t iter; + + im = NULL; + ium = NULL; + inm = NULL; + ims = NULL; + + /* + * Request headers land in r->inline_fields and only spill into the + * r->fields list, so a list-only walk silently sees nothing on an + * ordinary request. Use the iterator that covers both + * (src/nxt_http_parse.h). + */ + + for (f = nxt_http_fields_first(&iter, r->inline_fields, + r->num_inline_fields, r->fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip) { + continue; + } + + switch (f->name_length) { + + case nxt_length("If-Match"): + if (nxt_strncasecmp(f->name, (u_char *) "If-Match", + nxt_length("If-Match")) == 0) + { + im = f; + } + + break; + + case nxt_length("If-None-Match"): + if (nxt_strncasecmp(f->name, (u_char *) "If-None-Match", + nxt_length("If-None-Match")) == 0) + { + inm = f; + } + + break; + + case nxt_length("If-Modified-Since"): + if (nxt_strncasecmp(f->name, (u_char *) "If-Modified-Since", + nxt_length("If-Modified-Since")) == 0) + { + ims = f; + } + + break; + + case nxt_length("If-Unmodified-Since"): + if (nxt_strncasecmp(f->name, (u_char *) "If-Unmodified-Since", + nxt_length("If-Unmodified-Since")) == 0) + { + ium = f; + } + + break; + + default: + break; + } + } + + if (im != NULL) { + value.start = im->value; + value.length = im->value_length; + + /* Sect. 13.1.1: If-Match uses the strong comparison function. */ + + if (!nxt_http_static_etag_match(&value, etag, 1)) { + return NXT_HTTP_PRECONDITION_FAILED; + } + + } else if (ium != NULL) { + date = nxt_time_parse(ium->value, ium->value_length); + + if (date != (nxt_time_t) -1 && mtime > date) { + return NXT_HTTP_PRECONDITION_FAILED; + } + } + + if (inm != NULL) { + value.start = inm->value; + value.length = inm->value_length; + + if (nxt_http_static_etag_match(&value, etag, 0)) { + return NXT_HTTP_NOT_MODIFIED; + } + + return NXT_HTTP_OK; + } + + if (ims != NULL) { + date = nxt_time_parse(ims->value, ims->value_length); + + /* + * An unparsable date is ignored rather than treated as an error. + * Sect. 13.1.3 makes "earlier than or equal to" the unmodified case; + * nginx offers an "exact" mode to survive a rollback to an older + * mtime, but the RFC comparison is what Unit implements. + */ + + if (date != (nxt_time_t) -1 && mtime <= date) { + return NXT_HTTP_NOT_MODIFIED; + } + } + + return NXT_HTTP_OK; +} + + +/* + * Matches an entity-tag against an If-Match or If-None-Match list. + * + * Unit's own tags are always strong, so the comparison functions differ only + * in how they treat a weak tag on the client's side: under the strong + * function (Sect. 8.8.3.2) a W/-prefixed tag never matches, under the weak + * one the prefix is stripped and the opaque tags are compared. + * + * "*" matches any existing representation, but only as the entire field + * value -- the grammar is "*" / #entity-tag, so it is not a list member. + */ + +static nxt_bool_t +nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, nxt_bool_t strong) +{ + u_char *p, *end, *start; + nxt_bool_t weak; + nxt_str_t tag; + + p = list->start; + end = p + list->length; + + while (p < end && (*p == ' ' || *p == '\t')) { + p++; + } + + if (end - p == 1 && *p == '*') { + return 1; + } + + while (p < end) { + + while (p < end && (*p == ' ' || *p == '\t' || *p == ',')) { + p++; + } + + if (p == end) { + break; + } + + weak = 0; + + if (end - p >= 2 && p[0] == 'W' && p[1] == '/') { + weak = 1; + p += 2; + } + + if (p == end || *p != '"') { + /* Not a valid entity-tag; skip to the next comma. */ + + while (p < end && *p != ',') { + p++; + } + + continue; + } + + start = p++; + + while (p < end && *p != '"') { + p++; + } + + if (p == end) { + break; + } + + p++; + + if (weak && strong) { + continue; + } + + tag.start = start; + tag.length = p - start; + + if (nxt_strstr_eq(&tag, etag)) { + return 1; + } + } + + return 0; +} + + static void nxt_http_static_next(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx, nxt_http_status_t status) diff --git a/test/test_static.py b/test/test_static.py index 8eaf1cf47..52d0089e6 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -144,32 +144,111 @@ def test_static_range_ignored(): assert 'Content-Range' not in resp['headers'], 'no Content-Range' -def test_static_conditional_ignored(): - # Conditional requests are not implemented; a matching validator must - # not produce a 304 (which would imply the body was withheld). +def test_static_conditional_etag(): etag = client.get(url='/index.html')['headers']['ETag'] - resp = client.get( - url='/index.html', - headers={ - 'Host': 'localhost', - 'If-None-Match': etag, - 'Connection': 'close', - }, - ) - assert resp['status'] == 200, 'if-none-match ignored' - assert resp['body'] == '0123456789', 'full body on matching etag' + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) - resp = client.get( - url='/index.html', - headers={ - 'Host': 'localhost', - 'If-Modified-Since': 'Thu, 01 Jan 2100 00:00:00 GMT', - 'Connection': 'close', - }, - ) - assert resp['status'] == 200, 'if-modified-since ignored' - assert resp['body'] == '0123456789', 'full body on future date' + for value in [etag, f'W/{etag}', '*', f'"nope", {etag}']: + resp = get(**{'If-None-Match': value}) + assert resp['status'] == 304, f'304 for {value}' + assert resp['body'] == '', f'no body for {value}' + assert resp['headers']['ETag'] == etag, f'ETag echoed for {value}' + assert ( + 'Content-Length' not in resp['headers'] + ), f'no Content-Length for {value}' + + resp = get(**{'If-None-Match': '"nope"'}) + assert resp['status'] == 200, 'mismatch sends the body' + assert resp['body'] == '0123456789', 'full body on mismatch' + + +def test_static_conditional_if_match(): + etag = client.get(url='/index.html')['headers']['ETag'] + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + resp = get(**{'If-Match': etag}) + assert resp['status'] == 200, 'matching If-Match serves the file' + assert resp['body'] == '0123456789', 'full body' + + assert get(**{'If-Match': '*'})['status'] == 200, '* matches' + assert get(**{'If-Match': '"nope"'})['status'] == 412, 'mismatch is 412' + + # RFC 9110 Sect. 13.1.1: If-Match uses the strong comparison function, + # so a weak tag never matches even when the opaque tags are equal. + assert get(**{'If-Match': f'W/{etag}'})['status'] == 412, 'weak tag' + + # Sect. 13.2.2 evaluation order: If-Match is checked before + # If-None-Match, so a failed If-Match is a 412, not a 304. + resp = get(**{'If-Match': '"nope"', 'If-None-Match': etag}) + assert resp['status'] == 412, 'If-Match evaluated first' + + +def test_static_conditional_unmodified_since(): + def get(value, **extra): + return client.get( + url='/index.html', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'If-Unmodified-Since': value, + **extra, + }, + ) + + assert get('Thu, 01 Jan 2100 00:00:00 GMT')['status'] == 200, 'unmodified' + assert get('Thu, 01 Jan 2000 00:00:00 GMT')['status'] == 412, 'modified' + assert get('not-a-date')['status'] == 200, 'unparsable date is ignored' + + etag = client.get(url='/index.html')['headers']['ETag'] + + # If-Match takes precedence over If-Unmodified-Since. + resp = get('Thu, 01 Jan 2000 00:00:00 GMT', **{'If-Match': etag}) + assert resp['status'] == 200, 'If-Match wins' + + +def test_static_conditional_modified_since(temp_dir): + last_modified = client.get(url='/index.html')['headers']['Last-Modified'] + + def get(value, **extra): + return client.get( + url='/index.html', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'If-Modified-Since': value, + **extra, + }, + ) + + resp = get('Thu, 01 Jan 2100 00:00:00 GMT') + assert resp['status'] == 304, 'not modified since a future date' + assert resp['body'] == '', 'no body' + + resp = get(last_modified) + assert resp['status'] == 304, 'not modified since its own mtime' + + resp = get('Thu, 01 Jan 2000 00:00:00 GMT') + assert resp['status'] == 200, 'modified since an old date' + assert resp['body'] == '0123456789', 'full body' + + resp = get('not-a-date') + assert resp['status'] == 200, 'unparsable date is ignored' + + # RFC 9110 Sect. 13.2.2: a present If-None-Match wins outright, even when + # it does not match, so this must send the body despite the future date. + resp = get('Thu, 01 Jan 2100 00:00:00 GMT', **{'If-None-Match': '"nope"'}) + assert resp['status'] == 200, 'If-None-Match takes precedence' + assert resp['body'] == '0123456789', 'full body' def test_static_last_modified_is_gmt(temp_dir): From 537b04215edc0ef1318a2fc836153088a645a5bd Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 04:53:13 +0200 Subject: [PATCH 151/391] static: combine repeated If-Match and If-None-Match field lines nxt_http_static_preconditions() kept a single field pointer per header and overwrote it on each hit, so only the LAST line of a repeated header was evaluated. RFC 9110 Sect. 5.3 makes repeated field lines equivalent to one line holding the comma-separated concatenation. The If-Match case refused legitimate requests. Measured on a 10-byte file: If-Match: "" If-Match: "other" -> 412, where the combined list "", "other" gives 200. If-None-Match had the mirror-image bug: a first line naming the current entity-tag was discarded and the response was a full 200 instead of 304. Both headers are "#entity-tag" lists whose members are OR'd, so evaluating every line and remembering whether any matched is exactly the concatenation the RFC asks for, without buffering the lines. The date headers keep their existing last-line-wins behaviour: repeating them is not meaningful and the RFC gives no combining rule for a non-list field. Found by an adversarial test pass. The new test sends genuinely separate field lines (the harness emits one line per list element, test/unit/http.py) and fails without this change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- src/nxt_http_static.c | 54 +++++++++++++++++++++++++++++-------------- test/test_static.py | 33 ++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 0f0848230..614669520 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -945,12 +945,24 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, { nxt_str_t value; nxt_time_t date; - nxt_http_field_t *f, *im, *ium, *inm, *ims; + nxt_bool_t im_seen, im_match, inm_seen, inm_match; + nxt_http_field_t *f, *ium, *ims; nxt_http_fields_iter_t iter; - im = NULL; + /* + * RFC 9110 Sect. 5.3: repeated field lines are equivalent to one line + * holding the comma-separated concatenation. If-Match and If-None-Match + * are both "#entity-tag" lists whose members are OR'd, so evaluating + * every line and remembering whether ANY of them matched is exactly that + * concatenation -- whereas keeping only the last line seen would refuse a + * legitimate request with 412 when an earlier If-Match line matched. + */ + + im_seen = 0; + im_match = 0; + inm_seen = 0; + inm_match = 0; ium = NULL; - inm = NULL; ims = NULL; /* @@ -975,7 +987,16 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, if (nxt_strncasecmp(f->name, (u_char *) "If-Match", nxt_length("If-Match")) == 0) { - im = f; + im_seen = 1; + + if (!im_match) { + value.start = f->value; + value.length = f->value_length; + + /* Sect. 13.1.1: If-Match compares strongly. */ + + im_match = nxt_http_static_etag_match(&value, etag, 1); + } } break; @@ -984,7 +1005,14 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, if (nxt_strncasecmp(f->name, (u_char *) "If-None-Match", nxt_length("If-None-Match")) == 0) { - inm = f; + inm_seen = 1; + + if (!inm_match) { + value.start = f->value; + value.length = f->value_length; + + inm_match = nxt_http_static_etag_match(&value, etag, 0); + } } break; @@ -1012,13 +1040,8 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, } } - if (im != NULL) { - value.start = im->value; - value.length = im->value_length; - - /* Sect. 13.1.1: If-Match uses the strong comparison function. */ - - if (!nxt_http_static_etag_match(&value, etag, 1)) { + if (im_seen) { + if (!im_match) { return NXT_HTTP_PRECONDITION_FAILED; } @@ -1030,11 +1053,8 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, } } - if (inm != NULL) { - value.start = inm->value; - value.length = inm->value_length; - - if (nxt_http_static_etag_match(&value, etag, 0)) { + if (inm_seen) { + if (inm_match) { return NXT_HTTP_NOT_MODIFIED; } diff --git a/test/test_static.py b/test/test_static.py index 52d0089e6..686bf92af 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -167,6 +167,39 @@ def get(**headers): assert resp['body'] == '0123456789', 'full body on mismatch' +def test_static_conditional_duplicate_field_lines(): + # RFC 9110 Sect. 5.3: repeated field lines are equivalent to one line + # holding the comma-separated concatenation. Keeping only the last line + # seen refused a legitimate request with 412 when an earlier If-Match + # line matched. + etag = client.get(url='/index.html')['headers']['ETag'] + + def get(header, values): + return client.get( + url='/index.html', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + header: values, + }, + ) + + for values in [[etag, '"other"'], ['"other"', etag]]: + assert ( + get('If-None-Match', values)['status'] == 304 + ), f'If-None-Match matches in any line: {values}' + assert ( + get('If-Match', values)['status'] == 200 + ), f'If-Match matches in any line: {values}' + + assert ( + get('If-None-Match', ['"a"', '"b"'])['status'] == 200 + ), 'no If-None-Match line matches' + assert ( + get('If-Match', ['"a"', '"b"'])['status'] == 412 + ), 'no If-Match line matches' + + def test_static_conditional_if_match(): etag = client.get(url='/index.html')['headers']['ETag'] From 60ee106ff79f3fe94f553fa8463c79b3c77f58e0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 14:16:27 +0200 Subject: [PATCH 152/391] docs(changes): record conditional request support in changes.xml The CHANGES entry landed with the feature; docs/changes.xml did not. Both are maintained together -- the release notes are generated from the XML. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- docs/changes.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/changes.xml b/docs/changes.xml index 09308b519..e0fdfce9a 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,20 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +conditional requests for static files. A "share" action now answers a GET +or HEAD carrying "If-None-Match" or "If-Modified-Since" with "304 Not +Modified" when the file's validators still match, so a client that already +holds the file is not sent it again. Unit emitted "ETag" and +"Last-Modified" before, but ignored both when they came back. "If-Match" +and "If-Unmodified-Since" are evaluated first and answer "412 Precondition +Failed" when they do not hold. Entity-tag comparison is weak for +"If-None-Match" and strong for "If-Match", and "*" matches any +representation, per RFC 9110. + + + a file descriptor sent to a port is no longer leaked when the message is From 0115e3e7201d019b5685e3c61850ae8fa2a622fd Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 15:46:35 +0200 Subject: [PATCH 153/391] static: an unacceptable representation outranks a precondition Review of #338 found two ordering bugs at the seams with existing response processing. Both reproduce; both are fixed here. 1. Preconditions ran before content negotiation, so a 304 or 412 displaced a 406 that the request had already earned. Measured on a compressible file with compression configured: Accept-Encoding: identity;q=0, *;q=0 -> 406 ... plus If-None-Match: * -> 304 (wrong) ... plus If-Match: "nope" -> 412 (wrong) RFC 9110 Sect. 13.2.1 evaluates preconditions only for a request that would otherwise succeed. The fix could not simply move nxt_http_comp_check_compression() ahead of the precondition test: that function decides acceptability AND applies the decision, adding Content-Encoding and initialising the compressor. A 304 would then advertise an encoding it does not send, and leak the compressor, which is torn down by the last deflate() call that a 304 never makes. So it is split in two -- nxt_http_comp_check_acceptable() decides and remembers, nxt_http_comp_apply_compression() applies -- and the static handler puts precondition evaluation between them. The application response path in nxt_router.c always sends a body and calls both together; its behaviour is unchanged. 2. Preconditions compared against the ETag Unit derives from the file, but "response_headers" can replace or remove that header before it is sent (nxt_http_set_headers.c), so the comparison was against a validator the client never received. With ETag configured as "release-42": ETag: "release-42" advertised on an ordinary GET If-Match: "release-42" -> 412 (its own tag refused) If-None-Match: "release-42" -> 200 (no 304 where one was due) The 412 is the damaging half: a legitimate conditional request denied. Policy: when the matched action overrides ETag or Last-Modified, do not evaluate preconditions at all. That loses the 304 for those configurations and is never wrong, whereas comparing against the internal tag answers a question the client did not ask. Only the header NAME is inspected -- the value may be a template, and resolving it here would move tstr queries ahead of where nxt_http_set_headers() runs them. Tests: 406 precedence and a 304 carrying no Content-Encoding, in a new test_static_compression.py (compression configuration is not reversible within one unitd, #167, so it needs --restart like test_php_compression.py); the validator override; conditional HEAD; and a 304 followed by a normal GET on the same kept-alive connection, since every existing conditional test sends "Connection: close" and so did not establish that claim. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- src/nxt_http.h | 1 + src/nxt_http_compression.c | 61 +++++++++++++++---- src/nxt_http_compression.h | 4 +- src/nxt_http_set_headers.c | 54 +++++++++++++++++ src/nxt_http_static.c | 80 ++++++++++++++++--------- src/nxt_router.c | 19 ++++-- test/test_static.py | 95 +++++++++++++++++++++++++++++ test/test_static_compression.py | 102 ++++++++++++++++++++++++++++++++ 8 files changed, 374 insertions(+), 42 deletions(-) create mode 100644 test/test_static_compression.py diff --git a/src/nxt_http.h b/src/nxt_http.h index d473a4c81..51864f314 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -489,6 +489,7 @@ nxt_int_t nxt_http_rewrite_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf); nxt_int_t nxt_http_rewrite(nxt_task_t *task, nxt_http_request_t *r); +nxt_bool_t nxt_http_set_headers_override_validators(nxt_http_request_t *r); nxt_int_t nxt_http_set_headers_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf); nxt_int_t nxt_http_set_headers(nxt_http_request_t *r); diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 08460e6fe..884e78dfc 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -69,6 +69,13 @@ struct nxt_http_comp_compressor_s { struct nxt_http_comp_ctx_s { nxt_uint_t idx; + /* + * The compressor nxt_http_comp_check_acceptable() chose, or -1 when the + * request will not be compressed. Kept apart from "idx", which is only + * set once the choice has actually been applied. + */ + nxt_int_t sel_idx; + nxt_off_t resp_clen; nxt_off_t clen_sent; @@ -596,18 +603,29 @@ nxt_http_comp_is_resp_content_encoded(const nxt_http_request_t *r) } +/* + * Decides whether an acceptable representation exists, and remembers which + * compressor would be used, without touching the response or allocating a + * compressor context. + * + * Separated from applying that decision because RFC 9110 Sect. 13.2.1 puts + * this ahead of precondition evaluation: a request that cannot be satisfied + * at all must be answered 406, not 304 or 412. The caller therefore asks + * this first, evaluates preconditions, and only then applies -- so a 304 + * neither carries a Content-Encoding header nor leaves an initialised + * compressor behind, which would leak, since the compressor is torn down by + * the last deflate() call and a 304 makes none. + */ + nxt_int_t -nxt_http_comp_check_compression(nxt_task_t *task, nxt_http_request_t *r) +nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) { - int err; - nxt_int_t ret, idx; - nxt_off_t min_len; - nxt_str_t accept_encoding, mime_type = {}; - nxt_router_conf_t *rtcf; - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - nxt_http_comp_compressor_t *compressor; + nxt_int_t ret, idx; + nxt_str_t accept_encoding, mime_type = {}; + nxt_router_conf_t *rtcf; + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1 }; + *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1, .sel_idx = -1 }; if (nxt_http_comp_nr_enabled_compressors == 0) { return NXT_OK; @@ -664,7 +682,30 @@ nxt_http_comp_check_compression(nxt_task_t *task, nxt_http_request_t *r) return NXT_HTTP_NOT_ACCEPTABLE; } - if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + ctx->sel_idx = idx; + + return NXT_OK; +} + + +/* + * Applies the decision nxt_http_comp_check_acceptable() reached: adds the + * Content-Encoding header and initialises the compressor. Call it only on a + * path that will actually send a body. + */ + +nxt_int_t +nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) +{ + int err; + nxt_int_t idx; + nxt_off_t min_len; + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_compressor_t *compressor; + + idx = ctx->sel_idx; + + if (idx == -1 || idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { return NXT_OK; } diff --git a/src/nxt_http_compression.h b/src/nxt_http_compression.h index 99af8a669..2638ed97d 100644 --- a/src/nxt_http_compression.h +++ b/src/nxt_http_compression.h @@ -97,7 +97,9 @@ extern nxt_int_t nxt_http_comp_compress_static_response(nxt_task_t *task, size_t static_buf_len, size_t *out_total); extern bool nxt_http_comp_wants_compression(void); extern bool nxt_http_comp_compressor_is_valid(const nxt_str_t *token); -extern nxt_int_t nxt_http_comp_check_compression(nxt_task_t *task, +extern nxt_int_t nxt_http_comp_check_acceptable(nxt_task_t *task, + nxt_http_request_t *r); +extern nxt_int_t nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r); extern nxt_int_t nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, const nxt_conf_value_t *comp_conf); diff --git a/src/nxt_http_set_headers.c b/src/nxt_http_set_headers.c index 049abe62d..2ac6c696e 100644 --- a/src/nxt_http_set_headers.c +++ b/src/nxt_http_set_headers.c @@ -14,6 +14,60 @@ typedef struct { } nxt_http_header_val_t; +/* + * Whether the matched action replaces or removes one of the validators the + * static handler generates. + * + * A conditional request has to be judged against the validator the client was + * actually given. When "response_headers" sets ETag or Last-Modified, the + * value Unit derives from the file is not what went out, so comparing against + * it answers the wrong question -- it refuses an If-Match carrying the tag the + * server itself advertised. The static handler asks this and declines to + * evaluate preconditions at all in that case, which loses the 304 but is never + * wrong. + * + * Only the name matters here. The value may be a template resolved per + * request, and resolving it this early would move tstr queries ahead of where + * nxt_http_set_headers() runs them. + */ + +nxt_bool_t +nxt_http_set_headers_override_validators(nxt_http_request_t *r) +{ + nxt_uint_t i, n; + nxt_http_action_t *action; + nxt_http_header_val_t *header; + + action = r->action; + + if (action == NULL || action->set_headers == NULL) { + return 0; + } + + header = action->set_headers->elts; + n = action->set_headers->nelts; + + for (i = 0; i < n; i++) { + if (header[i].name.length == nxt_length("ETag") + && nxt_strncasecmp(header[i].name.start, (u_char *) "ETag", + nxt_length("ETag")) == 0) + { + return 1; + } + + if (header[i].name.length == nxt_length("Last-Modified") + && nxt_strncasecmp(header[i].name.start, + (u_char *) "Last-Modified", + nxt_length("Last-Modified")) == 0) + { + return 1; + } + } + + return 0; +} + + nxt_int_t nxt_http_set_headers_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf) diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 614669520..c5eb06873 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -739,6 +739,44 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, etag.start = field->value; etag.length = field->value_length; + if (exten.start == NULL) { + nxt_http_static_extract_extension(shr, &exten); + } + + if (mtype == NULL) { + mtype = nxt_http_static_mtype_get(&rtcf->mtypes_hash, &exten); + } + + if (mtype->length != 0) { + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { + goto fail; + } + + nxt_http_field_name_set(field, "Content-Type"); + + field->value = mtype->start; + field->value_length = mtype->length; + } + + r->resp.mime_type = mtype; + + /* + * RFC 9110 Sect. 13.2.1: an ordinary failure outranks a precondition. + * If no acceptable representation exists the answer is 406, and it + * must not be displaced by the 304 or 412 a validator would give -- + * so ask about acceptability here, and apply the decision further + * down, on the path that actually sends a body. + */ + + ret = nxt_http_comp_check_acceptable(task, r); + if (ret == NXT_HTTP_NOT_ACCEPTABLE) { + nxt_http_request_error(task, r, NXT_HTTP_NOT_ACCEPTABLE); + return; + } else if (ret != NXT_OK) { + goto fail; + } + pcond = nxt_http_static_preconditions(r, &etag, nxt_file_mtime(&fi)); if (pcond != NXT_HTTP_OK) { @@ -765,34 +803,9 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, goto send; } - if (exten.start == NULL) { - nxt_http_static_extract_extension(shr, &exten); - } - - if (mtype == NULL) { - mtype = nxt_http_static_mtype_get(&rtcf->mtypes_hash, &exten); - } - - if (mtype->length != 0) { - field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); - if (nxt_slow_path(field == NULL)) { - goto fail; - } - - nxt_http_field_name_set(field, "Content-Type"); - - field->value = mtype->start; - field->value_length = mtype->length; - } - - r->resp.mime_type = mtype; - if (ctx->need_body && nxt_file_size(&fi) > 0) { - ret = nxt_http_comp_check_compression(task, r); - if (ret == NXT_HTTP_NOT_ACCEPTABLE) { - nxt_http_request_error(task, r, NXT_HTTP_NOT_ACCEPTABLE); - return; - } else if (ret != NXT_OK) { + ret = nxt_http_comp_apply_compression(task, r); + if (nxt_slow_path(ret != NXT_OK)) { goto fail; } @@ -958,6 +971,19 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, * legitimate request with 412 when an earlier If-Match line matched. */ + /* + * Judge a conditional request only against validators the client was + * actually given. If "response_headers" replaces or removes ETag or + * Last-Modified, what this function would compare is not what went out + * (src/nxt_http_set_headers.c), so decline rather than answer 304 or 412 + * on the strength of a tag the client never saw. Serving the full + * response is always a correct answer to a conditional request. + */ + + if (nxt_http_set_headers_override_validators(r)) { + return NXT_HTTP_OK; + } + im_seen = 0; im_match = 0; inm_seen = 0; diff --git a/src/nxt_router.c b/src/nxt_router.c index 888a564a5..0610c7206 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5435,11 +5435,22 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, /* * Check compression before handing the chain over to r->out, so that - * the rejection below still owns it and releases it inline. - * nxt_http_comp_check_compression() reads r->resp and the request - * configuration, never r->out, so the order is free. + * the rejection below still owns it and releases it inline. Both + * calls read r->resp and the request configuration, never r->out, so + * the order is free. + * + * An application response is always sent with a body, so the two + * halves run together here. They are separate calls because the + * static path has to put precondition evaluation between them: a 406 + * outranks a precondition, but a 304 must not leave an initialised + * compressor behind (RFC 9110 Sect. 13.2.1). */ - ret = nxt_http_comp_check_compression(task, r); + ret = nxt_http_comp_check_acceptable(task, r); + if (ret != NXT_OK) { + goto fail; + } + + ret = nxt_http_comp_apply_compression(task, r); if (ret != NXT_OK) { goto fail; } diff --git a/test/test_static.py b/test/test_static.py index 686bf92af..0910d672b 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -249,6 +249,101 @@ def get(value, **extra): assert resp['status'] == 200, 'If-Match wins' +def test_static_conditional_validator_override(temp_dir): + # A conditional request is judged against the validator the client was + # given. When "response_headers" replaces ETag, the tag Unit derives + # from the file is not what went out, so preconditions are not evaluated + # at all -- previously an If-Match carrying the advertised tag was + # refused with 412, which is a legitimate request denied. + assets_dir = f'{temp_dir}/assets' + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + { + "action": { + "share": f'{assets_dir}$uri', + "response_headers": {"ETag": '"release-42"'}, + } + } + ], + } + ), 'override configure' + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + assert get()['headers']['ETag'] == '"release-42"', 'override advertised' + + assert ( + get(**{'If-Match': '"release-42"'})['status'] == 200 + ), 'the advertised tag is not refused' + assert ( + get(**{'If-None-Match': '"release-42"'})['status'] == 200 + ), 'no 304 against a tag Unit did not generate' + assert ( + get(**{'If-Match': '"nope"'})['status'] == 200 + ), 'and no 412 either, rather than a wrong one' + + +def test_static_conditional_keepalive(): + # The other conditional tests all send "Connection: close", so none of + # them establishes that a 304 leaves the connection usable. A 304 is + # bodyless with no Content-Length and no Transfer-Encoding, so a client + # that miscounts the framing would desynchronise here rather than fail + # outright. + etag = client.get(url='/index.html')['headers']['ETag'] + + resp, sock = client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'keep-alive', + 'If-None-Match': etag}, + start=True, + read_timeout=1, + ) + + assert resp['status'] == 304, '304 on a kept-alive connection' + assert resp['body'] == '', 'no body' + assert 'Content-Length' not in resp['headers'], 'no Content-Length' + assert 'Transfer-Encoding' not in resp['headers'], 'not chunked' + + resp = client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close'}, + sock=sock, + ) + + assert resp['status'] == 200, 'the same connection still serves' + assert resp['body'] == '0123456789', 'and the body is intact' + + +def test_static_conditional_head(): + etag = client.get(url='/index.html')['headers']['ETag'] + + def head(**headers): + return client.head( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + resp = head(**{'If-None-Match': etag}) + assert resp['status'] == 304, 'HEAD honours a matching validator' + assert resp['body'] == '', 'no body' + + resp = head(**{'If-None-Match': '"nope"'}) + assert resp['status'] == 200, 'HEAD with a stale validator' + assert resp['body'] == '', 'still no body on a HEAD' + assert ( + resp['headers']['Content-Length'] == '10' + ), 'HEAD reports the length it would have sent' + + assert head(**{'If-Match': '"nope"'})['status'] == 412, 'HEAD 412' + + def test_static_conditional_modified_since(temp_dir): last_modified = client.get(url='/index.html')['headers']['Last-Modified'] diff --git a/test/test_static_compression.py b/test/test_static_compression.py new file mode 100644 index 000000000..c8a1a43f1 --- /dev/null +++ b/test/test_static_compression.py @@ -0,0 +1,102 @@ +from pathlib import Path + +import pytest + +from unit.applications.proto import ApplicationProto +from unit.option import option + +client = ApplicationProto() + + +@pytest.fixture(autouse=True) +def requires_restart_mode(): + """ + Configuring compression is not reversible within one unitd (#167), so + these tests need their own unitd or they crash whichever test runs next. + Same reasoning as test_php_compression.py; drop both once #167 is fixed. + """ + if not option.restart: + pytest.skip('needs --restart until #167 is fixed') + + +@pytest.fixture(autouse=True) +def setup_method_fixture(temp_dir): + assets_dir = f'{temp_dir}/assets' + Path(assets_dir).mkdir(parents=True, exist_ok=True) + Path(f'{assets_dir}/big.css').write_text( + 'body{color:red}' * 500, encoding='utf-8' + ) + + assert 'success' in client.conf( + { + "settings": { + "http": { + "compression": { + "types": ["text/css"], + "compressors": [ + {"encoding": "gzip", "level": 5, "min_length": 10} + ], + } + } + }, + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"share": f'{assets_dir}$uri'}}], + } + ), 'compression configure' + + +def test_static_compression_baseline(): + resp = client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Accept-Encoding': 'gzip', + 'Connection': 'close', + }, + ) + assert resp['status'] == 200, 'compressed 200' + assert resp['headers']['Content-Encoding'] == 'gzip', 'gzip applied' + + +def test_static_compression_precondition_does_not_mask_406(): + # RFC 9110 Sect. 13.2.1: an ordinary failure outranks a precondition. A + # client that refuses every encoding cannot be served at all, so the + # answer is 406 -- a validator must not turn that into 304 or 412. + etag = client.get(url='/big.css')['headers']['ETag'] + + def get(**headers): + return client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'identity;q=0, *;q=0', + **headers, + }, + ) + + assert get()['status'] == 406, 'unacceptable without a validator' + assert get(**{'If-None-Match': '*'})['status'] == 406, '406 outranks 304' + assert get(**{'If-None-Match': etag})['status'] == 406, '406 over 304' + assert get(**{'If-Match': '"nope"'})['status'] == 406, '406 outranks 412' + + +def test_static_compression_304_carries_no_encoding(): + # A 304 sends no body, so it must not claim one is encoded, and the + # compressor must not have been initialised for it either. + etag = client.get(url='/big.css')['headers']['ETag'] + + resp = client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'gzip', + 'If-None-Match': etag, + }, + ) + + assert resp['status'] == 304, 'not modified' + assert resp['body'] == '', 'no body' + assert 'Content-Encoding' not in resp['headers'], 'no Content-Encoding' + assert 'Content-Length' not in resp['headers'], 'no Content-Length' From f916d259c764aec0cc7dafdb916abd4572b68f71 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 22:38:01 +0200 Subject: [PATCH 154/391] unitctl: stop parsing hjson unitctl pulled in nu-json to read hjson. That crate brings a Nushell utility layer with it: 30 crates in the tree of a tool that speaks one JSON control API. No document in the repository mentions hjson outside the --help string. Stdin with no file name used hjson for /config. It now uses JSON. hjson is a superset of JSON, so piping JSON is unchanged; comments and unquoted keys stop working. ".hjson" and ".cjson" files are still recognised, and refused with a message that names hjson. The alternative, letting them fall through to the JSON parser, reports a syntax error on the first comment and never says why. nu-json also switched serde_json's preserve_order on for the whole graph. unit-client-rs asks for that feature itself and needs it, so the feature stays; it now has one source instead of two. Measured with cargo tree --edges normal: 172 crates before, 142 after. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/Cargo.lock | 422 +------------------------ tools/unitctl/unitctl/Cargo.toml | 1 - tools/unitctl/unitctl/src/inputfile.rs | 93 +++--- tools/unitctl/unitctl/src/main.rs | 1 - tools/unitctl/unitctl/src/unitctl.rs | 2 +- 5 files changed, 39 insertions(+), 480 deletions(-) diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 4e00da7cc..5512edf9c 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -61,24 +61,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - [[package]] name = "aws-lc-rs" version = "1.17.1" @@ -108,21 +96,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bitflags" version = "2.13.1" @@ -172,51 +145,12 @@ dependencies = [ "serde_repr", ] -[[package]] -name = "buf-trait" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eaafc770e8c073d6c3facafe7617e774305d4954aa6351b9c452eb37ee17b4" -dependencies = [ - "zerocopy 0.7.35", -] - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "byteyarn" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93e51d26468a15ea59f8525e0c13dc405db43e644a0b1e6d44346c72cf4cf7b" -dependencies = [ - "buf-trait", -] - -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - [[package]] name = "cc" version = "1.2.67" @@ -235,12 +169,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - [[package]] name = "clap" version = "4.6.2" @@ -338,43 +266,12 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - [[package]] name = "custom_error" version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f8a51dd197fa6ba5b4dc98a990a43cc13693c23eb0089ebb0fcc1f04152bca6" -[[package]] -name = "derive_more" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" -dependencies = [ - "proc-macro2", - "quote", - "rustc_version", - "syn", - "unicode-xid", -] - [[package]] name = "dispatch2" version = "0.3.1" @@ -418,17 +315,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "fancy-regex" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fastrand" version = "2.4.1" @@ -875,17 +761,6 @@ dependencies = [ "libc", ] -[[package]] -name = "js-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - [[package]] name = "json5" version = "1.3.1" @@ -896,33 +771,12 @@ dependencies = [ "ucd-trie", ] -[[package]] -name = "lean_string" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9d7a64297827f9ef2cc8a111ca1d87bead398b7f9fe76ba0eb9f50d175e8d" -dependencies = [ - "castaway", - "itoa", - "ryu", - "serde_core", -] - [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" -dependencies = [ - "serde", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -935,31 +789,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - [[package]] name = "log" version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" -[[package]] -name = "lscolors" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d60e266dfb1426eb2d24792602e041131fdc0236bb7007abc0e589acafd60929" -dependencies = [ - "aho-corasick", - "nu-ansi-term", -] - [[package]] name = "memchr" version = "2.8.3" @@ -977,18 +812,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "ntapi" version = "0.4.3" @@ -998,72 +821,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "nu-json" -version = "0.113.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664096864c9fc825f7da6185bcc3b288869aedfe8d70580d41fa96f90661d71" -dependencies = [ - "linked-hash-map", - "nu-utils", - "num-traits", - "serde", - "serde_json", -] - -[[package]] -name = "nu-utils" -version = "0.113.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec685be6fcc963735a023948a38e7e39e03a32fedc09649c30019ce8f85fbfc8" -dependencies = [ - "byteyarn", - "crossterm_winapi", - "derive_more", - "fancy-regex", - "lean_string", - "log", - "lscolors", - "memchr", - "nix", - "num-format", - "parking_lot", - "serde", - "serde_json", - "strip-ansi-escapes", - "sys-locale", - "unicase", - "web-time", -] - -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - [[package]] name = "objc2" version = "0.6.4" @@ -1139,29 +896,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - [[package]] name = "pbr" version = "1.1.1" @@ -1206,7 +940,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.54", + "zerocopy", ] [[package]] @@ -1268,15 +1002,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.13.1" @@ -1320,15 +1045,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "1.1.4" @@ -1390,12 +1106,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - [[package]] name = "ryu" version = "1.0.23" @@ -1420,12 +1130,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "security-framework" version = "3.7.0" @@ -1449,12 +1153,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - [[package]] name = "serde" version = "1.0.228" @@ -1569,15 +1267,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "strip-ansi-escapes" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" -dependencies = [ - "vte", -] - [[package]] name = "strsim" version = "0.11.1" @@ -1612,15 +1301,6 @@ dependencies = [ "syn", ] -[[package]] -name = "sys-locale" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" -dependencies = [ - "libc", -] - [[package]] name = "sysinfo" version = "0.39.6" @@ -1776,24 +1456,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "unit-client-rs" version = "1.36.1" @@ -1848,7 +1516,6 @@ dependencies = [ "http-body-util", "hyper", "json5", - "nu-json", "rustls-pki-types", "serde", "serde_json", @@ -1897,15 +1564,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "vte" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" -dependencies = [ - "memchr", -] - [[package]] name = "walkdir" version = "2.5.0" @@ -1940,61 +1598,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "wasm-bindgen" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.126" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "webpki-roots" version = "1.0.8" @@ -2287,34 +1890,13 @@ dependencies = [ "synstructure", ] -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - [[package]] name = "zerocopy" version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ - "zerocopy-derive 0.8.54", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "zerocopy-derive", ] [[package]] diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index 0f00592de..eab68d1c2 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -17,7 +17,6 @@ clap = { version = "4.4", features = ["default", "derive", "cargo"] } custom_error = "1.9" serde = "1.0" json5 = "1.3.1" -nu-json = "0.113.0" serde_json = { version = "1.0", optional = false } serde_yaml = "0.9" # The "std" feature is not on by default and pem::from_buf needs it. diff --git a/tools/unitctl/unitctl/src/inputfile.rs b/tools/unitctl/unitctl/src/inputfile.rs index 6f2083de0..d0fac0023 100644 --- a/tools/unitctl/unitctl/src/inputfile.rs +++ b/tools/unitctl/unitctl/src/inputfile.rs @@ -14,6 +14,9 @@ pub enum InputFormat { Yaml, Json, Json5, + /// Recognised, but not parsed. unitctl no longer reads hjson. The format + /// stays in this list so that an ".hjson" file gets a message saying that, + /// instead of a JSON parse error. Hjson, Pem, JavaScript, @@ -51,7 +54,7 @@ impl InputFormat { .split_once('/') .map_or(lead_slash_removed, |(first, _)| first); match first_path { - "config" => InputFormat::Hjson, + "config" => InputFormat::Json, "certificates" => InputFormat::Pem, "js_modules" => InputFormat::JavaScript, _ => InputFormat::Json, @@ -138,11 +141,6 @@ impl InputFile { } } - /// Converts a HJSON Value type to a JSON Value type - fn hjson_value_to_json_value(value: nu_json::Value) -> serde_json::Value { - serde_json::to_value(value).expect("Failed to convert HJSON value to JSON value") - } - pub fn to_unit_serializable_map(&self) -> Result { let reader: Box = self.try_into()?; let body_data: UnitSerializableMap = match self.format() { @@ -159,23 +157,12 @@ impl InputFile { json5::from_str(&json5_string) .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })? } - InputFormat::Hjson => { - // nu_json::Map and not HashMap: with the crate's preserve_order - // feature, on by default, it is a LinkedHashMap and keeps the - // document's member order. A HashMap here shuffled the members - // of every hjson input, which is the same defect the response - // side of this fixes. - let hjson_value: nu_json::Map = nu_json::from_reader(reader) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?; - - hjson_value - .iter() - .map(|(k, v)| { - let json_value = Self::hjson_value_to_json_value(v.clone()); - (k.clone(), json_value) - }) - .collect() - } + // Refuse hjson by name. Handing the file to the JSON parser would + // report a syntax error on the first comment or unquoted key, and + // that error does not tell the user what to do. + InputFormat::Hjson => Err(UnitctlError::DeserializationError { + message: "hjson is no longer supported: convert the file to JSON first".to_string(), + })?, _ => Err(UnitctlError::DeserializationError { message: format!("Unsupported input format for serialization: {:?}", self), })?, @@ -254,6 +241,8 @@ mod tests { assert_eq!(InputFormat::from_file_extension("yml"), InputFormat::Yaml); assert_eq!(InputFormat::from_file_extension("json"), InputFormat::Json); assert_eq!(InputFormat::from_file_extension("json5"), InputFormat::Json5); + assert_eq!(InputFormat::from_file_extension("hjson"), InputFormat::Hjson); + assert_eq!(InputFormat::from_file_extension("cjson"), InputFormat::Hjson); assert_eq!(InputFormat::from_file_extension("pem"), InputFormat::Pem); assert_eq!(InputFormat::from_file_extension("js"), InputFormat::JavaScript); assert_eq!(InputFormat::from_file_extension("njs"), InputFormat::JavaScript); @@ -262,14 +251,14 @@ mod tests { #[test] fn can_parse_remote_paths() { - assert_eq!(InputFormat::from_remote_path("//config"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("/config"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("/config/"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("config/"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("config"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("/config/something/"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("config/something/"), InputFormat::Hjson); - assert_eq!(InputFormat::from_remote_path("config/something"), InputFormat::Hjson); + assert_eq!(InputFormat::from_remote_path("//config"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("/config"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("/config/"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("config/"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("config"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("/config/something/"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("config/something/"), InputFormat::Json); + assert_eq!(InputFormat::from_remote_path("config/something"), InputFormat::Json); assert_eq!(InputFormat::from_remote_path("/certificates"), InputFormat::Pem); assert_eq!(InputFormat::from_remote_path("/certificates/"), InputFormat::Pem); assert_eq!(InputFormat::from_remote_path("certificates/"), InputFormat::Pem); @@ -322,9 +311,9 @@ mod tests { } /// `UnitSerializableMap` is a `serde_json::Map`, which keeps member order - /// only because every format feeding it keeps it too. `serde_yaml`, `json5` - /// and the hjson `collect()` each reach the type by a different route, so - /// each one is checked. + /// only because every format feeding it keeps it too. `serde_yaml` and + /// `json5` each reach the type by a different route, so each one is + /// checked. #[test] fn every_input_format_keeps_member_order() { let json = MEMBERS @@ -336,7 +325,6 @@ mod tests { assert_eq!(members_of(&write_input(&json, ".json"), InputFormat::Json), MEMBERS); assert_eq!(members_of(&write_input(&json, ".json5"), InputFormat::Json5), MEMBERS); - assert_eq!(members_of(&write_input(&json, ".hjson"), InputFormat::Hjson), MEMBERS); let yaml = MEMBERS .iter() @@ -345,30 +333,21 @@ mod tests { assert_eq!(members_of(&write_input(&yaml, ".yaml"), InputFormat::Yaml), MEMBERS); } - /// The values have to arrive intact, not just the names: hjson reaches - /// `serde_json::Value` through a conversion of its own. + /// hjson is refused by name. Handing the file to the JSON parser instead + /// would report a syntax error on the first comment, which does not tell + /// the user what to do. #[test] - fn an_hjson_input_keeps_its_values_and_nesting() { - let file = write_input( - "{\n listeners: {\n \"*:8080\": { pass: \"routes\", tls: { certificate: \"bundle\", session: {} } }\n }\n routes: [ { action: { return: 204 } } ]\n}\n", - ".hjson", - ); - let map = InputFile::FileWithFormat(file.path().into(), InputFormat::Hjson) + fn an_hjson_input_is_refused_with_a_message_about_hjson() { + let file = write_input("{\n # a comment\n routes: []\n}\n", ".hjson"); + let error = InputFile::from(file.path()) .to_unit_serializable_map() - .expect("hjson must deserialize"); - - assert_eq!(map.keys().cloned().collect::>(), vec!["listeners", "routes"]); - assert_eq!(map["listeners"]["*:8080"]["pass"], "routes"); - assert_eq!(map["routes"][0]["action"]["return"], 204); - - // Nested order is the other half of this, and it holds only while both - // crates keep preserve_order: every nu_json::Value goes through - // serde_json::to_value on the way in. So pin the nested names too, in - // an order no map sorts into by accident. - let listener = map["listeners"]["*:8080"].as_object().expect("a listener"); - assert_eq!(listener.keys().cloned().collect::>(), vec!["pass", "tls"]); - let tls = listener["tls"].as_object().expect("a tls object"); - assert_eq!(tls.keys().cloned().collect::>(), vec!["certificate", "session"]); + .expect_err("hjson must be refused"); + + assert!( + error.to_string().contains("hjson is no longer supported"), + "unexpected message: {}", + error + ); } /// The way back in is strict, which is why `unitctl export` warns that the diff --git a/tools/unitctl/unitctl/src/main.rs b/tools/unitctl/unitctl/src/main.rs index b7a779fde..71e719bf6 100644 --- a/tools/unitctl/unitctl/src/main.rs +++ b/tools/unitctl/unitctl/src/main.rs @@ -1,7 +1,6 @@ extern crate clap; extern crate colored_json; extern crate custom_error; -extern crate nu_json; extern crate serde; extern crate unit_client_rs; diff --git a/tools/unitctl/unitctl/src/unitctl.rs b/tools/unitctl/unitctl/src/unitctl.rs index 43f2b7776..cb5f43da0 100644 --- a/tools/unitctl/unitctl/src/unitctl.rs +++ b/tools/unitctl/unitctl/src/unitctl.rs @@ -84,7 +84,7 @@ pub(crate) enum Commands { global = true, short = 'f', long = "file", - help = "Input file (json, json5, cjson, hjson yaml, pem) to send to unit when applicable use - for stdin" + help = "Input file (json, json5, yaml, pem) to send to unit when applicable use - for stdin" )] input_file: Option, From 79a1bab1d5f55836a7bbd99a86666aa18a8e30d4 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 22:43:42 +0200 Subject: [PATCH 155/391] docs(changes): unitctl no longer reads hjson ddde8ee4 (step6-drop-hjson) drops nu-json from unitctl. Document the two user-visible effects: stdin to /config is now parsed as JSON, and .hjson/.cjson files are refused by name instead of hitting a misleading JSON parse error. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 11 +++++++++++ docs/changes.xml | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/CHANGES b/CHANGES index ce5a1043a..40c094864 100644 --- a/CHANGES +++ b/CHANGES @@ -135,6 +135,17 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 must now speak HTTP/1.1 to unitctl; one that only accepts HTTP/2 can no longer be reached. + *) Change: unitctl no longer reads hjson. Input piped to "/config" on + stdin is parsed as JSON now, not hjson; hjson is a superset of JSON, + so piping JSON still works, but comments and unquoted keys do not. A + ".hjson" or ".cjson" file is refused with "hjson is no longer + supported: convert the file to JSON first", instead of being handed + to the JSON parser, which reports a misleading error at the first + comment. Supported input formats are now json, json5, yaml and + pem. This removes the nu-json dependency and the 30 crates of + Nushell utility code it pulled in for a tool that speaks one JSON + control API. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index e0fdfce9a..ad205824b 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -185,6 +185,19 @@ HTTP/1.1 to unitctl; one that only accepts HTTP/2 can no longer be reached. + + +unitctl no longer reads hjson. Input piped to "/config" on stdin is parsed as +JSON now, not hjson; hjson is a superset of JSON, so piping JSON still works, +but comments and unquoted keys do not. A ".hjson" or ".cjson" file is refused +with "hjson is no longer supported: convert the file to JSON first", instead of +being handed to the JSON parser, which reports a misleading error at the first +comment. Supported input formats are now json, json5, yaml and pem. This +removes the nu-json dependency and the 30 crates of Nushell utility code it +pulled in for a tool that speaks one JSON control API. + + + From 29069ae2556eac3b42281c8ba5ae656c8949bfab Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 23:51:47 +0200 Subject: [PATCH 156/391] static: close the file when no representation is acceptable A share with compression configured answers 406 when the client refuses every coding it could offer. That exit returns without closing the file it has already opened, while every other bodyless exit in the function -- the 304, 412 and 416 branches, and "fail:" -- closes first. So one unauthenticated request leaks one descriptor: Accept-Encoding: identity;q=0, *;q=0 Measured on a compressible file with gzip configured: ten such requests left ten descriptors open on it in /proc, and ten ordinary 200s left none. Repeating it exhausts the router's table, which stops every application behind that router. This is not new in the unreleased tree. The same "return" without a close is in 1.35.4, 1.35.5, 1.36.0 and 1.36.1 -- every release since compression was added -- so it is a fix to shipped code, not to work in flight. An earlier reading of mine, and a review, both put its origin in the conditional-request work that reorganised this function; checking the tags shows the branch predates it and the reorganisation only moved it. Verified after the fix: twenty such requests, zero descriptors held, and the 406 itself unchanged. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- CHANGES | 16 ++++++++++++++++ docs/changes.xml | 24 ++++++++++++++++++++++++ src/nxt_http_static.c | 10 ++++++++++ 3 files changed, 50 insertions(+) diff --git a/CHANGES b/CHANGES index 40c094864..35464c1c7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Security: a file descriptor is no longer leaked when a static request + cannot be satisfied by any acceptable representation. A share with + compression configured answered "406 Not Acceptable" without closing + the file it had opened, so a single unauthenticated request carrying + "Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, + and repeating it exhausted the table. + *) Feature: conditional requests for static files. A "share" action now answers a GET or HEAD carrying "If-None-Match" or "If-Modified-Since" with "304 Not Modified" when the file's validators still match, so a @@ -12,6 +19,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 "If-Unmodified-Since" are evaluated first and answer "412 Precondition Failed" when they do not hold. + *) Security: a file descriptor is no longer leaked when a static request + cannot be satisfied by any acceptable representation. A share with + compression configured answered "406 Not Acceptable" without closing + the file it had opened, so a single unauthenticated request carrying + "Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, + and repeating it exhausted the table, stopping every application + behind that router. Present since compression was added: 1.35.4, + 1.35.5, 1.36.0 and 1.36.1 are all affected. + *) Security: a file descriptor sent to a port is no longer leaked when the message is dispatched and nothing takes ownership of it. An application holds the write end of the router's main port and the diff --git a/docs/changes.xml b/docs/changes.xml index ad205824b..77595784c 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,17 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +a file descriptor is no longer leaked when a static request cannot be +satisfied by any acceptable representation. A share with compression +configured answered "406 Not Acceptable" without closing the file it had +opened, so a single unauthenticated request carrying +"Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, and +repeating it exhausted the table. + + + conditional requests for static files. A "share" action now answers a GET @@ -24,6 +35,19 @@ representation, per RFC 9110. + + +a file descriptor is no longer leaked when a static request cannot be +satisfied by any acceptable representation. A share with compression +configured answered "406 Not Acceptable" without closing the file it had +opened, so a single unauthenticated request carrying +"Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, and +repeating it exhausted the table, stopping every application behind that +router. Present since compression was added: 1.35.4, 1.35.5, 1.36.0 and +1.36.1 are all affected. + + + a file descriptor sent to a port is no longer leaked when the message is diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index c5eb06873..b18551be4 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -771,6 +771,16 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, ret = nxt_http_comp_check_acceptable(task, r); if (ret == NXT_HTTP_NOT_ACCEPTABLE) { + /* + * Every other exit that answers without a body closes the file + * first -- the 304, 412 and 416 branches below, and "fail:". + * This one returns without reaching any of them, so it has to + * close its own, or one unauthenticated request costs the router + * a descriptor. + */ + nxt_file_close(task, f); + f = NULL; + nxt_http_request_error(task, r, NXT_HTTP_NOT_ACCEPTABLE); return; } else if (ret != NXT_OK) { From af7f7a076083c3e8f689a856a7d7242878bbdced Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 23:52:27 +0200 Subject: [PATCH 157/391] http: weaken the entity-tag of a coded representation Unit builds a static file's ETag from its mtime and size, so the gzip response and the identity response advertise the same strong validator. RFC 9110 Sect. 8.8.3 requires a strong validator to change whenever the selected representation changes, and a content coding selects a different representation. Inert until something compares strongly. If-Range does: Sect. 13.1.5 mandates the strong comparison, so a client that took its tag from a gzip response, then resumed with that tag and a Range, matched -- and the range path serves identity bytes, because it skips compression deliberately. The client appends identity bytes at offsets it believes are gzip offsets and its saved file is silently corrupt. Download managers and "curl -C -" do exactly this; browsers do not. Weaken the tag when a coding is applied, in nxt_http_comp_apply_compression() where the Content-Encoding header is added. The strong comparison then fails, so If-Range is ignored and the whole representation is sent. If-None-Match is unaffected: it compares weakly, so revalidating a coded representation still answers 304. All three servers checked do this, differing only in how: nginx weakens with ngx_http_weak_etag() from its gzip header filter, Apache appends "-gzip" inside the quotes, and Go's net/http appends the coding name. Weakening is the smallest of the three and needs no allocation beyond the prefix. Measured, gzip configured for text/css: identity gives "6aa71a84-1d4c" and gzip gives W/"6aa71a84-1d4c"; If-Range with the gzip tag now answers 200 where it previously answered 206, If-Range with the identity tag still answers 206, and If-None-Match with the gzip tag still answers 304. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- CHANGES | 17 +++++++++ docs/changes.xml | 23 +++++++++++++ src/nxt_http_compression.c | 70 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/CHANGES b/CHANGES index 35464c1c7..01ce4a8ab 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: a compressed static response no longer carries the same strong + entity-tag as the uncompressed one. Unit derives the tag from the + file's mtime and size, which do not change with the content coding, so + both representations advertised an identical strong validator. RFC 9110 + requires a strong validator to change whenever the selected + representation changes; the tag of a coded representation is now weak, + as nginx does. + *) Security: a file descriptor is no longer leaked when a static request cannot be satisfied by any acceptable representation. A share with compression configured answered "406 Not Acceptable" without closing @@ -19,6 +27,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 "If-Unmodified-Since" are evaluated first and answer "412 Precondition Failed" when they do not hold. + *) Change: a content-coded response now carries a weak entity-tag, so + "If-Range" cannot resume a coded representation; "If-None-Match" + revalidation is unaffected. Unit derives the tag from the file's mtime + and size, which do not change with the content coding, so a gzip + response and the identity one previously advertised the same strong + validator -- 1.36.1 and earlier shipped that. RFC 9110 requires a + strong validator to change whenever the selected representation + changes, and nginx weakens the tag for the same reason. + *) Security: a file descriptor is no longer leaked when a static request cannot be satisfied by any acceptable representation. A share with compression configured answered "406 Not Acceptable" without closing diff --git a/docs/changes.xml b/docs/changes.xml index 77595784c..cd4314b21 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,17 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +a compressed static response no longer carries the same strong entity-tag as +the uncompressed one. Unit derives the tag from the file's mtime and size, +which do not change with the content coding, so both representations +advertised an identical strong validator. RFC 9110 requires a strong +validator to change whenever the selected representation changes; the tag of +a coded representation is now weak, as nginx does. + + + a file descriptor is no longer leaked when a static request cannot be @@ -35,6 +46,18 @@ representation, per RFC 9110. + + +a content-coded response now carries a weak entity-tag, so "If-Range" cannot +resume a coded representation; "If-None-Match" revalidation is unaffected. +Unit derives the tag from the file's mtime and size, which do not change with +the content coding, so a gzip response and the identity one previously +advertised the same strong validator -- 1.36.1 and earlier shipped that. +RFC 9110 requires a strong validator to change whenever the selected +representation changes, and nginx weakens the tag for the same reason. + + + a file descriptor is no longer leaked when a static request cannot be diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 884e78dfc..d2d659cfd 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -688,6 +688,72 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) } +/* + * Weakens the response's ETag, so that a coded representation does not carry + * the same strong validator as the identity one. + * + * RFC 9110 Sect. 8.8.3: a strong validator must change whenever the selected + * representation changes, and a content coding selects a different + * representation. Unit derives the tag from the file's mtime and size, which + * are the same whichever coding is served, so without this a client that took + * its tag from a gzip response and then sent it back in If-Range -- which + * Sect. 13.1.5 compares strongly -- would match, and be handed identity bytes + * for offsets it believes are gzip offsets. Weakening makes that strong + * comparison fail, so the range is simply not applied. + * + * If-None-Match keeps working: it compares weakly, so revalidation of a coded + * representation still answers 304. + * + * nginx does the same thing (ngx_http_weak_etag(), called from its gzip + * header filter); Apache appends "-gzip" and Go's net/http appends the coding + * name. All three make the tag differ per coding; weakening is the smallest + * of the three and needs no extra allocation beyond the prefix. + */ + +static nxt_int_t +nxt_http_comp_weaken_etag(nxt_http_request_t *r) +{ + u_char *p; + nxt_http_field_t *f; + nxt_http_fields_iter_t iter; + + for (f = nxt_http_fields_first(&iter, r->resp.inline_fields, + r->resp.num_inline_fields, r->resp.fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip || f->name_length != nxt_length("ETag") + || nxt_strncasecmp(f->name, (u_char *) "ETag", + nxt_length("ETag")) != 0) + { + continue; + } + + if (f->value_length >= 2 + && f->value[0] == 'W' && f->value[1] == '/') + { + return NXT_OK; + } + + p = nxt_mp_nget(r->mem_pool, f->value_length + nxt_length("W/")); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + /* Copy out of the old value before the field is repointed. */ + nxt_memcpy(p, "W/", nxt_length("W/")); + nxt_memcpy(p + nxt_length("W/"), f->value, f->value_length); + + f->value = p; + f->value_length += nxt_length("W/"); + + return NXT_OK; + } + + return NXT_OK; +} + + /* * Applies the decision nxt_http_comp_check_acceptable() reached: adds the * Content-Encoding header and initialises the compressor. Call it only on a @@ -726,6 +792,10 @@ nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) nxt_http_comp_set_header(r, idx); + if (nxt_slow_path(nxt_http_comp_weaken_etag(r) != NXT_OK)) { + return NXT_ERROR; + } + ctx->idx = idx; ctx->ctx.level = nxt_http_comp_enabled_compressors[idx].opts.level; From a8ddab24840504fd923fd3ff9ebf91c82f50949e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 23:53:04 +0200 Subject: [PATCH 158/391] static: implement HTTP byte-range requests A "share" action now answers a single-range "Range" request with 206 and a "Content-Range" header, an unsatisfiable one with 416, and advertises support with "Accept-Ranges: bytes". "If-Range" is honoured, so a range is served only while the client's copy is still current. The send path already served an arbitrary window of a file: nxt_http_static_body_handler() works from fb->file_pos to fb->file_end and nxt_http_static_buf_completion() reads positionally, so serving bytes A..B is file_pos = A, file_end = B + 1. There is no sendfile(2) path in the static handler and the 206 and 416 status lines were already in the h1 tables, so the router is untouched -- contrary to the 2023 assessment on nginx/unit#1011 that this needed "medium changes to the Unit Router". Deliberately single-range only. A multi-range request falls back to a full 200, which a server may always do, and that sidesteps the whole CVE-2011-3192 class: Apache had to add overlap and reversal counting with caps of 200 ranges, 20 overlaps and 20 reversals, and Go caps the summed range size against the file size, purely to bound multipart output. None of that is needed while only one range is served. Compression is skipped for a satisfiable range: coding a byte slice would either compress the wrong bytes or require re-deriving which coded bytes correspond to the requested identity range. Serving the range from identity is what nginx does in effect, by disabling ranges entirely once its gzip filter runs. A HEAD is range-processed exactly like the GET it stands for and differs only in sending no body. Sect. 14.2 reads like an argument for ignoring Range on a HEAD, and that was tried; but Sect. 9.3.2 asks a HEAD to send the headers the GET would have sent, and nginx, Apache and Go's net/http all answer 206 with Content-Range to a HEAD, with "curl -I -r" relying on it. Two defects found while building it, folded in rather than left as history: *) A suffix range against a zero-length file answered 206 with the malformed "Content-Range: bytes 0--1/0". Sect. 14.4 makes any range unsatisfiable against a zero-length representation. *) The range number parser accumulated digits unbounded, and a value that wrapped negative passed both bounds tests: "rest" then went negative, nxt_min() cast it to a huge size_t, the buffer allocation failed and the request was abandoned with the file still open. "bytes=9223372036854775808-" returned 206 with a negative Content-Range and leaked one descriptor per request. Values now saturate at NXT_OFF_T_MAX, which is also the RFC answer at both ends. The Content-Range buffer is sized for three %O conversions rather than two. With two, 40 digits are shared among start, end and size: 13 each fits, 14 does not, so a file of about 10 TB with a range near its end truncated the header. nxt_sprintf() clamps at the boundary, so the value was cut rather than the buffer overrun -- a well-formed but wrong header, which for a resuming client is worse than a crash. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- CHANGES | 28 +-- docs/changes.xml | 26 +-- src/nxt_http.h | 2 + src/nxt_http_static.c | 385 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 389 insertions(+), 52 deletions(-) diff --git a/CHANGES b/CHANGES index 01ce4a8ab..aa6cdf594 100644 --- a/CHANGES +++ b/CHANGES @@ -1,20 +1,20 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 - *) Bugfix: a compressed static response no longer carries the same strong - entity-tag as the uncompressed one. Unit derives the tag from the - file's mtime and size, which do not change with the content coding, so - both representations advertised an identical strong validator. RFC 9110 - requires a strong validator to change whenever the selected - representation changes; the tag of a coded representation is now weak, - as nginx does. - - *) Security: a file descriptor is no longer leaked when a static request - cannot be satisfied by any acceptable representation. A share with - compression configured answered "406 Not Acceptable" without closing - the file it had opened, so a single unauthenticated request carrying - "Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, - and repeating it exhausted the table. + *) Feature: byte-range requests for static files. A "share" action now + answers a satisfiable single-range "Range: bytes=..." request with + "206 Partial Content" and a matching "Content-Range", and an + unsatisfiable one with "416 Range Not Satisfiable"; "Accept-Ranges: + bytes" is now sent on a plain 200 as well. The three single-range + forms are supported -- "A-B", "A-" and the suffix form "-N" -- with + the end clamped to the last byte of the file; a multi-range request + is ignored and answered with the full 200, as RFC 9110 permits. + "If-Range" is honoured against the file's entity-tag (strong + comparison) or its "Last-Modified" date, falling back to the full + 200 on a mismatch. Preconditions (RFC 9110 Sect. 13.2.2) are still + evaluated first, so a 304 or 412 outcome wins over any Range. A + response served from a Range is not content-coded; compression + still applies to a full 200. *) Feature: conditional requests for static files. A "share" action now answers a GET or HEAD carrying "If-None-Match" or "If-Modified-Since" diff --git a/docs/changes.xml b/docs/changes.xml index cd4314b21..73b2f4e49 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,25 +10,15 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> - - -a compressed static response no longer carries the same strong entity-tag as -the uncompressed one. Unit derives the tag from the file's mtime and size, -which do not change with the content coding, so both representations -advertised an identical strong validator. RFC 9110 requires a strong -validator to change whenever the selected representation changes; the tag of -a coded representation is now weak, as nginx does. - - - - + -a file descriptor is no longer leaked when a static request cannot be -satisfied by any acceptable representation. A share with compression -configured answered "406 Not Acceptable" without closing the file it had -opened, so a single unauthenticated request carrying -"Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, and -repeating it exhausted the table. +byte-range requests for static files. A "share" action answers a "Range" +header naming a single range with "206 Partial Content" and a +"Content-Range" header, an unsatisfiable one with "416 Range Not +Satisfiable", and advertises support with "Accept-Ranges: bytes". +"If-Range" is honoured, so a range is served only while the client's copy +is still current. A request naming several ranges is answered in full, as +a server that does not implement multipart ranges may. diff --git a/src/nxt_http.h b/src/nxt_http.h index 51864f314..82ddd1498 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -19,6 +19,7 @@ typedef enum { NXT_HTTP_SWITCHING_PROTOCOLS = 101, NXT_HTTP_OK = 200, + NXT_HTTP_PARTIAL_CONTENT = 206, NXT_HTTP_NO_CONTENT = 204, NXT_HTTP_MULTIPLE_CHOICES = 300, @@ -39,6 +40,7 @@ typedef enum { NXT_HTTP_PRECONDITION_FAILED = 412, NXT_HTTP_PAYLOAD_TOO_LARGE = 413, NXT_HTTP_URI_TOO_LONG = 414, + NXT_HTTP_RANGE_NOT_SATISFIABLE = 416, NXT_HTTP_UPGRADE_REQUIRED = 426, NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE = 431, diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index b18551be4..6ddc6f6ff 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -171,6 +171,9 @@ static nxt_http_status_t nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, nxt_time_t mtime); static nxt_bool_t nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, nxt_bool_t strong); +static nxt_http_status_t nxt_http_static_range(nxt_http_request_t *r, + nxt_str_t *etag, nxt_time_t mtime, nxt_off_t size, nxt_off_t *start, + nxt_off_t *end); #if (NXT_HAVE_OPENAT2) static u_char *nxt_http_static_chroot_match(u_char *chr, u_char *shr); #endif @@ -493,6 +496,9 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_uint_t level; nxt_file_t *f, file; nxt_file_info_t fi; + nxt_off_t range_start, range_end; + nxt_bool_t is_range; + nxt_http_status_t rstatus; nxt_http_field_t *field; nxt_http_status_t status, pcond; nxt_router_conf_t *rtcf; @@ -773,10 +779,9 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, if (ret == NXT_HTTP_NOT_ACCEPTABLE) { /* * Every other exit that answers without a body closes the file - * first -- the 304, 412 and 416 branches below, and "fail:". - * This one returns without reaching any of them, so it has to - * close its own, or one unauthenticated request costs the router - * a descriptor. + * first -- the 304 and 416 branches below, and "fail:". This one + * returns rather than reaching either, so it has to close its own, + * or one unauthenticated request costs the router a descriptor. */ nxt_file_close(task, f); f = NULL; @@ -813,30 +818,135 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, goto send; } - if (ctx->need_body && nxt_file_size(&fi) > 0) { - ret = nxt_http_comp_apply_compression(task, r); - if (nxt_slow_path(ret != NXT_OK)) { + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { + goto fail; + } + + nxt_http_field_name_set(field, "Accept-Ranges"); + + field->value = (u_char *) "bytes"; + field->value_length = nxt_length("bytes"); + + /* + * A HEAD is range-processed exactly like the GET it stands for, and + * differs only in sending no body. RFC 9110 Sect. 14.2 says range + * handling is defined for GET, which reads like an argument for + * ignoring Range here -- but Sect. 9.3.2 asks a HEAD to send the same + * header fields the GET would have sent, and nginx, Apache and Go's + * net/http all answer 206 with Content-Range to a HEAD. "curl -I -r" + * relies on it. Matching them is worth more than the stricter + * reading of a sentence about methods that do not define ranges. + */ + + rstatus = nxt_http_static_range(r, &etag, nxt_file_mtime(&fi), + nxt_file_size(&fi), &range_start, + &range_end); + + if (rstatus == NXT_HTTP_RANGE_NOT_SATISFIABLE) { + nxt_file_close(task, f); + f = NULL; + + r->status = NXT_HTTP_RANGE_NOT_SATISFIABLE; + r->resp.content_length_n = 0; + + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { goto fail; } - if (nxt_http_comp_wants_compression()) { - size_t out_total; - nxt_int_t ret; + nxt_http_field_name_set(field, "Content-Range"); - ret = nxt_http_comp_compress_static_response( - task, r, &f, &fi, - NXT_HTTP_STATIC_BUF_SIZE, - &out_total); - if (ret == NXT_ERROR) { - goto fail; - } + length = nxt_length("bytes */") + NXT_OFF_T_LEN; + + p = nxt_mp_nget(r->mem_pool, length); + if (nxt_slow_path(p == NULL)) { + goto fail; + } + + field->value = p; + field->value_length = nxt_sprintf(p, p + length, "bytes */%O", + nxt_file_size(&fi)) + - p; + + body_handler = NULL; + goto send; + } + + is_range = (rstatus == NXT_HTTP_PARTIAL_CONTENT); + + if (is_range) { + r->status = NXT_HTTP_PARTIAL_CONTENT; + r->resp.content_length_n = range_end - range_start + 1; + + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { + goto fail; + } + + nxt_http_field_name_set(field, "Content-Range"); + + /* + * Three %O conversions, so three times NXT_OFF_T_LEN. With two, + * 40 digits are available across start, end and size: 13 digits + * each still fits, 14 does not, so a file of about 10 TB with a + * range near its end truncates the header. nxt_sprintf() clamps + * at the boundary, so the value is cut rather than the buffer + * overrun -- a wrong Content-Range rather than a crash. + */ + length = nxt_length("bytes -/") + 3 * NXT_OFF_T_LEN; + + p = nxt_mp_nget(r->mem_pool, length); + if (nxt_slow_path(p == NULL)) { + goto fail; + } + + field->value = p; + field->value_length = nxt_sprintf(p, p + length, + "bytes %O-%O/%O", range_start, + range_end, nxt_file_size(&fi)) + - p; + } - ret = nxt_file_info(f, &fi); + if (ctx->need_body && nxt_file_size(&fi) > 0) { + + /* + * A satisfiable Range request is served as identity partial + * content: content-coding a byte slice would either compress + * the wrong bytes (coding the whole file, then slicing, defeats + * the point of a range request) or require re-deriving which + * coded bytes correspond to the requested identity range, which + * most codings do not support at all. Skipping compression + * here is a plain read of the file, so it does not touch the + * temp-file swap that nxt_http_comp_compress_static_response() + * performs, nor r->resp.mime_type (already set above). + */ + + if (!is_range) { + ret = nxt_http_comp_apply_compression(task, r); if (nxt_slow_path(ret != NXT_OK)) { goto fail; } - r->resp.content_length_n = out_total; + if (nxt_http_comp_wants_compression()) { + size_t out_total; + nxt_int_t ret; + + ret = nxt_http_comp_compress_static_response( + task, r, &f, &fi, + NXT_HTTP_STATIC_BUF_SIZE, + &out_total); + if (ret == NXT_ERROR) { + goto fail; + } + + ret = nxt_file_info(f, &fi); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + + r->resp.content_length_n = out_total; + } } fb = nxt_http_static_buf_alloc(task, r->mem_pool); @@ -845,7 +955,14 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, } fb->file = f; - fb->file_end = nxt_file_size(&fi); + + if (is_range) { + fb->file_pos = range_start; + fb->file_end = range_end + 1; + + } else { + fb->file_end = nxt_file_size(&fi); + } r->out = fb; @@ -1201,6 +1318,234 @@ nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, nxt_bool_t strong) } +/* + * Parses a decimal run of digits starting at *p (bounded by end), advancing + * *p past what it consumed. Returns the parsed value, or -1 if *p pointed + * at a non-digit (no digits consumed, *p left unchanged). + */ + +static nxt_off_t +nxt_http_static_range_number(u_char **p, u_char *end) +{ + u_char *start; + nxt_off_t value; + + start = *p; + + if (*p == end || **p < '0' || **p > '9') { + return -1; + } + + value = 0; + + while (*p < end && **p >= '0' && **p <= '9') { + /* + * Saturate rather than wrap. A wrapped value goes NEGATIVE, and a + * negative first-pos passes both the "a >= size" and "b < a" tests, + * so the range is accepted and "rest = file_end - file_pos" in + * nxt_http_static_body_handler() comes out negative: nxt_min() casts + * it to a huge size_t, the buffer allocation fails, and the request + * is abandoned with the file still open in r->out. One header per + * leaked descriptor is an unauthenticated denial of service. + * + * Saturating is also what the RFC asks for at both ends: a first-pos + * of NXT_OFF_T_MAX is >= size, so Sect. 14.1.2 gives 416, while a + * suffix that large means "the whole representation". + */ + + if (value > (NXT_OFF_T_MAX - (**p - '0')) / 10) { + value = NXT_OFF_T_MAX; + + while (*p < end && **p >= '0' && **p <= '9') { + (*p)++; + } + + return value; + } + + value = value * 10 + (*(*p)++ - '0'); + } + + if (*p == start) { + return -1; + } + + return value; +} + + +/* + * RFC 9110 Sect. 14.1-14.4: parses a "Range" request header and, when + * "If-Range" (Sect. 13.1.5) is present, applies it only if the precondition + * matches the current representation. + * + * Returns: + * NXT_HTTP_OK -- no Range applies; serve the full 200. + * (No Range header, a malformed Range, a multi-range request -- a + * server may legally ignore Range entirely -- or an If-Range mismatch.) + * NXT_HTTP_PARTIAL_CONTENT -- "start"/"end" name an inclusive byte range + * to serve as a 206; both are clamped to [0, size - 1]. + * NXT_HTTP_RANGE_NOT_SATISFIABLE -- the single range is out of bounds; the + * caller answers 416 with a "Content-Range: bytes STAR/size" header. + */ + +static nxt_http_status_t +nxt_http_static_range(nxt_http_request_t *r, nxt_str_t *etag, + nxt_time_t mtime, nxt_off_t size, nxt_off_t *start, nxt_off_t *end) +{ + u_char *p, *last; + nxt_off_t a, b, suffix; + nxt_time_t date; + nxt_bool_t match; + nxt_str_t value; + nxt_http_field_t *f, *range, *if_range; + nxt_http_fields_iter_t iter; + + range = NULL; + if_range = NULL; + + for (f = nxt_http_fields_first(&iter, r->inline_fields, + r->num_inline_fields, r->fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip) { + continue; + } + + switch (f->name_length) { + + case nxt_length("Range"): + if (nxt_strncasecmp(f->name, (u_char *) "Range", + nxt_length("Range")) == 0) + { + range = f; + } + + break; + + case nxt_length("If-Range"): + if (nxt_strncasecmp(f->name, (u_char *) "If-Range", + nxt_length("If-Range")) == 0) + { + if_range = f; + } + + break; + + default: + break; + } + } + + if (range == NULL) { + return NXT_HTTP_OK; + } + + if (if_range != NULL) { + value.start = if_range->value; + value.length = if_range->value_length; + + if (value.length > 0 && (value.start[0] == '"' + || (value.length > 1 + && value.start[0] == 'W' + && value.start[1] == '/'))) + { + /* An entity-tag: Sect. 13.1.5 requires the strong comparison. */ + + match = nxt_http_static_etag_match(&value, etag, 1); + + } else { + date = nxt_time_parse(value.start, value.length); + + /* Sect. 13.1.5: an exact match against the last modification. */ + + match = (date != (nxt_time_t) -1 && date == mtime); + } + + if (!match) { + return NXT_HTTP_OK; + } + } + + p = range->value; + last = p + range->value_length; + + if ((size_t) (last - p) <= nxt_length("bytes=") + || nxt_strncasecmp(p, (u_char *) "bytes=", nxt_length("bytes=")) != 0) + { + return NXT_HTTP_OK; + } + + p += nxt_length("bytes="); + + /* + * Only a single range-spec is supported; a comma anywhere in the + * remainder marks a multi-range request, which a server may ignore. + */ + + if (memchr(p, ',', last - p) != NULL) { + return NXT_HTTP_OK; + } + + if (*p == '-') { + p++; + + suffix = nxt_http_static_range_number(&p, last); + + if (suffix == -1 || p != last) { + return NXT_HTTP_OK; + } + + /* + * A suffix range is unsatisfiable when it asks for nothing, and also + * against a zero-length representation: "size - suffix" would clamp + * to 0 while "size - 1" is -1, yielding "Content-Range: bytes 0--1/0". + */ + + if (suffix == 0 || size == 0) { + return NXT_HTTP_RANGE_NOT_SATISFIABLE; + } + + a = (suffix < size) ? size - suffix : 0; + b = size - 1; + + } else { + a = nxt_http_static_range_number(&p, last); + + if (a == -1 || p == last || *p != '-') { + return NXT_HTTP_OK; + } + + p++; + + if (a >= size) { + return NXT_HTTP_RANGE_NOT_SATISFIABLE; + } + + if (p == last) { + b = size - 1; + + } else { + b = nxt_http_static_range_number(&p, last); + + if (b == -1 || p != last || b < a) { + return NXT_HTTP_OK; + } + + if (b >= size) { + b = size - 1; + } + } + } + + *start = a; + *end = b; + + return NXT_HTTP_PARTIAL_CONTENT; +} + + static void nxt_http_static_next(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx, nxt_http_status_t status) From af733f12a885ea24f34b674188e692e573e3e1e1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 23:53:20 +0200 Subject: [PATCH 159/391] test(static): adversarial probes for conditional and byte-range requests Written to break the feature rather than confirm it, after its own tests were green. They found two defects, both folded into the implementation commit: the zero-length suffix range, and the wrapped range value that leaked a descriptor per request. Covers numeric overflow at both ends, ranges spanning the 128 KiB buffer boundary with byte-exact slice assertions, 24 threads against one file proving no stale file_pos leaks through the per-thread buffer freelist, the full Sect. 13.2.2 precedence matrix, HEAD, Accept-Encoding interaction, and the share features -- index, fallback, types, directory redirect -- being unaffected by a validator or a Range. Every 206 and 416 asserts its Content-Range, not just its status. That is the lesson from the zero-length bug: it survived a fully green suite because the tests checked status and body length, and a 206 whose Content-Length is 0 looks benign until the header itself is read. test_static_range_gzip needs --restart because compression configuration is not reversible within one unitd (freeunitorg/freeunit#167); it is skipped otherwise. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- test/test_static.py | 1070 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1058 insertions(+), 12 deletions(-) diff --git a/test/test_static.py b/test/test_static.py index 0910d672b..117e628a6 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -127,23 +127,154 @@ def test_static_etag(temp_dir): assert etag != client.get(url='/')['headers']['ETag'], 'new ETag' -def test_static_range_ignored(): - # Unit does not implement byte-range requests for static files. Per - # RFC 7233 a server that does not support Range MUST ignore it and - # return the full 200 representation -- never a malformed 206. - resp = client.get( +def test_static_accept_ranges(): + resp = client.get(url='/index.html') + assert resp['status'] == 200, 'plain 200' + assert resp['headers']['Accept-Ranges'] == 'bytes', 'Accept-Ranges on 200' + + +def range_get(**headers): + return client.get( url='/index.html', - headers={ - 'Host': 'localhost', - 'Range': 'bytes=0-4', - 'Connection': 'close', - }, + headers={'Host': 'localhost', 'Connection': 'close', **headers}, ) - assert resp['status'] == 200, 'range ignored -> full 200' - assert resp['body'] == '0123456789', 'full body returned' + + +def test_static_range_satisfiable(): + # index.html is the 10-byte "0123456789". + resp = range_get(Range='bytes=0-4') + assert resp['status'] == 206, '0-4 is 206' + assert resp['body'] == '01234', '0-4 body slice' + assert resp['headers']['Content-Range'] == 'bytes 0-4/10' + assert resp['headers']['Content-Length'] == '5' + assert resp['headers']['Accept-Ranges'] == 'bytes' + + resp = range_get(Range='bytes=5-') + assert resp['status'] == 206, '5- is 206' + assert resp['body'] == '56789', '5- body slice' + assert resp['headers']['Content-Range'] == 'bytes 5-9/10' + + resp = range_get(Range='bytes=-5') + assert resp['status'] == 206, '-5 is 206' + assert resp['body'] == '56789', '-5 (suffix) body slice' + assert resp['headers']['Content-Range'] == 'bytes 5-9/10' + + # "0-" (whole file) is a valid, satisfiable single range -> 206. + resp = range_get(Range='bytes=0-') + assert resp['status'] == 206, '0- is 206' + assert resp['body'] == '0123456789', '0- body is the whole file' + assert resp['headers']['Content-Range'] == 'bytes 0-9/10' + + # Past EOF but still satisfiable: the end clamps to the last byte. + resp = range_get(Range='bytes=5-9999') + assert resp['status'] == 206, '5-9999 clamps and is 206' + assert resp['body'] == '56789', '5-9999 body slice' + assert resp['headers']['Content-Range'] == 'bytes 5-9/10' + + +def test_static_range_unsatisfiable(): + resp = range_get(Range='bytes=10-') + assert resp['status'] == 416, 'start at EOF is 416' + assert resp['body'] == '', 'no body on 416' + assert resp['headers']['Content-Range'] == 'bytes */10' + + # "-0" (a zero-length suffix) is unsatisfiable, unlike "0-". + resp = range_get(Range='bytes=-0') + assert resp['status'] == 416, '-0 is unsatisfiable' + assert resp['headers']['Content-Range'] == 'bytes */10' + + +def test_static_range_empty_file(temp_dir): + # RFC 9110 Sect. 14.4: no range is satisfiable against a zero-length + # representation. The suffix form is the trap: "size - suffix" clamps to + # 0 while "size - 1" is -1, which produced a 206 carrying the malformed + # "Content-Range: bytes 0--1/0". + Path(f'{temp_dir}/assets/empty.txt').write_text('', encoding='utf-8') + + def get(**headers): + return client.get( + url='/empty.txt', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + for value in ['bytes=-5', 'bytes=0-', 'bytes=0-4']: + resp = get(**{'Range': value}) + assert resp['status'] == 416, f'416 for {value} on an empty file' + assert ( + resp['headers']['Content-Range'] == 'bytes */0' + ), f'unsatisfiable Content-Range for {value}' + + resp = get() + assert resp['status'] == 200, 'no Range still serves the empty file' + assert resp['body'] == '', 'empty body' + + +def test_static_range_malformed(): + for value in ['bytes=abc', 'bytes=', 'bytes=5-1', 'notbytes=0-4']: + resp = range_get(Range=value) + assert resp['status'] == 200, f'malformed {value!r} -> full 200' + assert resp['body'] == '0123456789', f'full body for {value!r}' + assert 'Content-Range' not in resp['headers'], f'no Content-Range for {value!r}' + + +def test_static_range_multi_ignored(): + # A server may legally ignore a multi-range request; only single ranges + # are implemented, so this must fall back to a full 200. + resp = range_get(Range='bytes=0-4,5-9') + assert resp['status'] == 200, 'multi-range falls back to 200' + assert resp['body'] == '0123456789', 'full body' assert 'Content-Range' not in resp['headers'], 'no Content-Range' +def test_static_range_if_range(): + r = client.get(url='/index.html') + etag = r['headers']['ETag'] + last_modified = r['headers']['Last-Modified'] + + # Matching tag: apply the range. + resp = range_get(Range='bytes=0-4', **{'If-Range': etag}) + assert resp['status'] == 206, 'If-Range matching tag applies range' + assert resp['body'] == '01234' + + # Non-matching tag: ignore Range, serve full 200. + resp = range_get(Range='bytes=0-4', **{'If-Range': '"nope"'}) + assert resp['status'] == 200, 'If-Range mismatched tag ignores range' + assert resp['body'] == '0123456789' + + # Matching date: apply the range. + resp = range_get(Range='bytes=0-4', **{'If-Range': last_modified}) + assert resp['status'] == 206, 'If-Range matching date applies range' + assert resp['body'] == '01234' + + # Non-matching (older) date: ignore Range, serve full 200. + resp = range_get( + Range='bytes=0-4', + **{'If-Range': 'Thu, 01 Jan 2000 00:00:00 GMT'}, + ) + assert resp['status'] == 200, 'If-Range mismatched date ignores range' + assert resp['body'] == '0123456789' + + # If-Range without Range is meaningless and must not affect the result. + resp = range_get(**{'If-Range': etag}) + assert resp['status'] == 200, 'If-Range without Range is a no-op' + assert resp['body'] == '0123456789' + + +def test_static_range_conditional_wins(): + etag = client.get(url='/index.html')['headers']['ETag'] + + # RFC 9110 Sect. 13.2.2: preconditions are evaluated before Range is + # considered at all -- a 304 (or 412) must win over a Range request. + resp = range_get(Range='bytes=0-4', **{'If-None-Match': etag}) + assert resp['status'] == 304, '304 wins over Range' + assert resp['body'] == '', 'no body on 304' + assert 'Content-Range' not in resp['headers'], 'no Content-Range on 304' + + resp = range_get(Range='bytes=0-4', **{'If-Match': '"nope"'}) + assert resp['status'] == 412, '412 wins over Range' + assert 'Content-Range' not in resp['headers'], 'no Content-Range on 412' + + def test_static_conditional_etag(): etag = client.get(url='/index.html')['headers']['ETag'] @@ -728,3 +859,918 @@ def one_request(): assert one_request() == b'0123456789', f'keep-alive request {i}' finally: sock.close() + + +# --------------------------------------------------------------------------- +# Adversarial probes for the conditional-request and byte-range features. +# Every 206 asserts its Content-Range, every 416 too: the earlier suite passed +# while a suffix range against an empty file emitted "bytes 0--1/0". +# --------------------------------------------------------------------------- + + +def _raw(url, header_lines, method='GET'): + """Send a request built from literal header lines (so duplicated header + fields and odd bytes survive) and return the parsed response.""" + req = f'{method} {url} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + req = req.encode() + for line in header_lines: + if isinstance(line, str): + line = line.encode() + req += line + b'\r\n' + req += b'\r\n' + + return client.http(req, raw=True, encoding='latin-1') + + +def test_static_range_header_syntax(): + # index.html is the 10-byte "0123456789". + # + # RFC 9110 Sect. 14.2: a server MAY ignore Range entirely, so a 200 is + # always legal; what is not legal is a 206 whose Content-Range and body + # disagree, or a 416 for a satisfiable range. + def check(value, body=None, crange=None): + resp = range_get(Range=value) + + if body is None: + assert resp['status'] == 200, f'{value!r} -> 200' + assert resp['body'] == '0123456789', f'{value!r} full body' + assert 'Content-Range' not in resp['headers'], value + return + + assert resp['status'] == 206, f'{value!r} -> 206' + assert resp['body'] == body, f'{value!r} body' + assert resp['headers']['Content-Range'] == crange, f'{value!r} range' + assert resp['headers']['Content-Length'] == str(len(body)), value + + # Sect. 14.1: range-unit names are case-insensitive. + check('BYTES=0-4', '01234', 'bytes 0-4/10') + check('Bytes=0-4', '01234', 'bytes 0-4/10') + + # Exactly one byte, first and last positions. + check('bytes=0-0', '0', 'bytes 0-0/10') + check('bytes=9-9', '9', 'bytes 9-9/10') + check('bytes=9-', '9', 'bytes 9-9/10') + check('bytes=-1', '9', 'bytes 9-9/10') + + # Whole file, several spellings; the end clamps to size-1 (Sect. 14.1.2). + check('bytes=0-9', '0123456789', 'bytes 0-9/10') + check('bytes=0-10', '0123456789', 'bytes 0-9/10') + check('bytes=-10', '0123456789', 'bytes 0-9/10') + check('bytes=-11', '0123456789', 'bytes 0-9/10') + check('bytes=-99999', '0123456789', 'bytes 0-9/10') + + # Leading zeros are plain decimal digits (1*DIGIT). + check('bytes=000-004', '01234', 'bytes 0-4/10') + check('bytes=-05', '56789', 'bytes 5-9/10') + + # The grammar has no OWS after "=" or around "-": ignoring is the + # RFC-blessed way out (Sect. 14.2), so a 200 is expected and a 206 must + # be self-consistent. + for value in ['bytes= 0-4', 'bytes=0 - 4', 'bytes=0-4 ', ' bytes=0-4', + 'bytes =0-4']: + resp = range_get(Range=value) + assert resp['status'] in (200, 206), f'{value!r} status' + if resp['status'] == 206: + assert resp['body'] == '01234', f'{value!r} body' + assert resp['headers']['Content-Range'] == 'bytes 0-4/10', value + else: + assert resp['body'] == '0123456789', f'{value!r} full body' + + # No unit, unknown unit, empty value, garbage. + for value in ['0-4', '=0-4', 'items=0-4', '', 'bytes', 'bytes=-', + 'bytes=--5', 'bytes=0--4', 'bytes=0-4-', 'bytes=+0-4', + 'bytes=-+4', 'bytes=0x0-4', 'bytes=,0-4', 'bytes=0-4,']: + check(value) + + +def test_static_range_last_byte_boundaries(): + # start == size-1 is the last satisfiable start, start == size is not. + resp = range_get(Range='bytes=9-100') + assert resp['status'] == 206 + assert resp['body'] == '9' + assert resp['headers']['Content-Range'] == 'bytes 9-9/10' + + resp = range_get(Range='bytes=10-10') + assert resp['status'] == 416, 'start == size is unsatisfiable' + assert resp['headers']['Content-Range'] == 'bytes */10' + assert resp['headers']['Content-Length'] == '0' + + resp = range_get(Range='bytes=11-') + assert resp['status'] == 416 + assert resp['headers']['Content-Range'] == 'bytes */10' + + +def test_static_range_one_byte_file(temp_dir): + Path(f'{temp_dir}/assets/one').write_text('x', encoding='utf-8') + + def get(**headers): + return client.get( + url='/one', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + for value in ['bytes=0-0', 'bytes=0-', 'bytes=-1', 'bytes=-2', + 'bytes=0-5']: + resp = get(Range=value) + assert resp['status'] == 206, f'{value} on 1-byte file' + assert resp['body'] == 'x', value + assert resp['headers']['Content-Range'] == 'bytes 0-0/1', value + assert resp['headers']['Content-Length'] == '1', value + + for value in ['bytes=1-', 'bytes=1-1', 'bytes=-0']: + resp = get(Range=value) + assert resp['status'] == 416, f'{value} on 1-byte file' + assert resp['headers']['Content-Range'] == 'bytes */1', value + assert resp['body'] == '' + + +def _check_range_honest(value, resp, size=10, content='0123456789'): + """A Range request may be answered 200 (ignored), 416 (unsatisfiable) or + 206 -- but a 206 must carry a well-formed Content-Range whose slice is + exactly the body.""" + status = resp['status'] + assert status in (200, 206, 416), f'{value!r} -> {status}' + + if status == 200: + assert resp['body'] == content, value + assert 'Content-Range' not in resp['headers'], value + + elif status == 416: + assert resp['headers']['Content-Range'] == f'bytes */{size}', value + assert resp['body'] == '' + + else: + crange = resp['headers']['Content-Range'] + assert crange.startswith('bytes ') and '--' not in crange, ( + f'{value!r}: malformed Content-Range {crange!r}' + ) + first, last = crange[6:].split('/')[0].split('-') + first, last = int(first), int(last) + assert 0 <= first <= last < size, f'{value!r}: {crange}' + assert resp['body'] == content[first : last + 1], value + assert resp['headers']['Content-Length'] == str(last - first + 1), ( + value + ) + + +def test_static_range_odd_bytes(): + # Control bytes inside the value: either the parser rejects the message + # (400) or the range is ignored/served honestly. Nothing may hang or + # emit a 206 whose slice does not match its Content-Range. + for value in [b'bytes=0-4\x00', b'bytes=\x000-4', b'bytes=0\x00-4', + b'bytes=0-4\x01', b'bytes=0-\x7f4', b'bytes=0-4\x0b', + b'bytes=0-4\x0c', b'bytes=\xff0-4', b'bytes=0-4\r', + b'bytes=0-4\t', b'\tbytes=0-4']: + resp = _raw('/index.html', [b'Range: ' + value]) + status = resp['status'] + assert status in (200, 206, 400), f'{value!r} -> {status}' + if status == 206: + assert resp['headers']['Content-Range'] == 'bytes 0-4/10', value + assert resp['body'] == '01234', value + elif status == 200: + assert resp['body'] == '0123456789', value + + # obs-fold continuation: either 400 or spliced as a single space. + resp = _raw('/index.html', ['Range: bytes=0-4', ' ,5-9']) + assert resp['status'] in (200, 206, 400) + if resp['status'] == 206: + assert resp['headers']['Content-Range'] == 'bytes 0-4/10' + assert resp['body'] == '01234' + + +def test_static_range_duplicate_headers(): + # Range is not a list-based field (RFC 9110 Sect. 14.2), so two Range + # lines make an invalid message: reject (400) or ignore (200). Serving a + # 206 for one of them is tolerated as long as it is self-consistent. + resp = _raw('/index.html', ['Range: bytes=0-4', 'Range: bytes=5-9']) + assert resp['status'] in (200, 206, 400) + if resp['status'] == 206: + crange = resp['headers']['Content-Range'] + assert crange in ('bytes 0-4/10', 'bytes 5-9/10'), crange + expected = '01234' if crange == 'bytes 0-4/10' else '56789' + assert resp['body'] == expected + + # Two If-Range lines, one matching, one not: the field is a single + # validator, so this is invalid; whichever is picked, the outcome must be + # a coherent 200 or 206. + etag = client.get(url='/index.html')['headers']['ETag'] + resp = _raw('/index.html', ['Range: bytes=0-4', f'If-Range: {etag}', + 'If-Range: "nope"']) + assert resp['status'] in (200, 206, 400) + if resp['status'] == 206: + assert resp['headers']['Content-Range'] == 'bytes 0-4/10' + assert resp['body'] == '01234' + + +def test_static_conditional_duplicate_list_headers(): + # If-None-Match and If-Match are list-based (#entity-tag), and RFC 9110 + # Sect. 5.3 lets a sender split a list across field lines; the recipient + # MUST treat them as the one combined list. The matching tag on the + # FIRST line must therefore count. + etag = client.get(url='/index.html')['headers']['ETag'] + + # Regression: only the last If-None-Match line used to be consulted. + resp = _raw('/index.html', [f'If-None-Match: {etag}', + 'If-None-Match: "other"']) + assert resp['status'] == 304, 'matching tag on the first INM line' + + resp = _raw('/index.html', ['If-None-Match: "other"', + f'If-None-Match: {etag}']) + assert resp['status'] == 304, 'matching tag on the last INM line' + + # Regression: only the last If-Match line used to be consulted, so a + # matching tag on the first line was lost and the request was refused. + resp = _raw('/index.html', [f'If-Match: {etag}', 'If-Match: "other"']) + assert resp['status'] == 200, 'matching tag on the first IM line' + assert resp['body'] == '0123456789' + + resp = _raw('/index.html', ['If-Match: "other"', f'If-Match: {etag}']) + assert resp['status'] == 200, 'matching tag on the last IM line' + + +def test_static_conditional_header_syntax(): + etag = client.get(url='/index.html')['headers']['ETag'] + opaque = etag.strip('"') + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + # Field names are case-insensitive. + assert get(**{'if-none-match': etag})['status'] == 304 + assert get(**{'IF-NONE-MATCH': etag})['status'] == 304 + assert get(**{'if-match': '"nope"'})['status'] == 412 + assert get(**{'RANGE': 'bytes=0-4'})['status'] == 206 + + # OWS and list punctuation around the tag (Sect. 5.6.1). + for value in [f' {etag}', f'{etag} ', f'\t{etag}', f'"x",{etag}', + f'"x", {etag}', f'{etag},"x"', f'"x" , {etag} , "y"', + f',{etag}', f'{etag},', f',,{etag},,']: + assert get(**{'If-None-Match': value})['status'] == 304, repr(value) + + # Weak tag vs. weak comparison (INM) and strong comparison (IM). + assert get(**{'If-None-Match': f'W/{etag}'})['status'] == 304 + assert get(**{'If-None-Match': f'"x", W/{etag}'})['status'] == 304 + assert get(**{'If-Match': f'W/{etag}'})['status'] == 412 + assert get(**{'If-Match': f'{etag}, W/"x"'})['status'] == 200 + + # "*" only as the entire field value. + assert get(**{'If-None-Match': '*'})['status'] == 304 + assert get(**{'If-None-Match': ' * '})['status'] == 304 + assert get(**{'If-Match': '*'})['status'] == 200 + assert get(**{'If-Match': '*, "x"'})['status'] == 412 + assert get(**{'If-None-Match': '"x", *'})['status'] == 200 + assert get(**{'If-None-Match': '**'})['status'] == 200 + assert get(**{'If-None-Match': '"*"'})['status'] == 200 + + # Tags that merely contain or are contained by ours must not match. + for value in [opaque, f'"{opaque}x"', f'"x{opaque}"', f'"{opaque[:-1]}"', + f'"{opaque.upper()}"', f'"{etag}']: + assert get(**{'If-None-Match': value})['status'] == 200, repr(value) + + # Unterminated quote must not run into a following list element. + assert get(**{'If-None-Match': f'"x, {etag}'})['status'] == 200 + + # Empty values: If-None-Match "" matches nothing -> 200. + assert get(**{'If-None-Match': ''})['status'] == 200 + assert get(**{'If-Range': ''})['status'] == 200 + + # A long list (just under the 8 KiB header limit) still finds the tag + # at the end / never matches. + long_list = ', '.join(['"%08d"' % i for i in range(500)]) + assert get(**{'If-None-Match': long_list + ', ' + etag})['status'] == 304 + assert get(**{'If-None-Match': long_list})['status'] == 200 + + +def test_static_conditional_dates(temp_dir): + mtime = int(time.time()) - 86400 * 3 + os.utime(f'{temp_dir}/assets/index.html', (mtime, mtime)) + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + last_modified = get()['headers']['Last-Modified'] + dt = parsedate_to_datetime(last_modified) + + # All three HTTP-date formats (Sect. 5.6.7) name the same instant. + rfc850 = dt.strftime('%A, %d-%b-%y %H:%M:%S GMT') + asctime = dt.strftime('%a %b ') + f'{dt.day:2d}' + dt.strftime( + ' %H:%M:%S %Y' + ) + + for value in [last_modified, rfc850, asctime]: + assert get(**{'If-Modified-Since': value})['status'] == 304, value + assert get(**{'If-Unmodified-Since': value})['status'] == 200, value + r = get(Range='bytes=0-4', **{'If-Range': value}) + assert r['status'] == 206, f'If-Range {value!r} exact' + assert r['headers']['Content-Range'] == 'bytes 0-4/10', value + + before = formatdate(mtime - 1, usegmt=True) + after = formatdate(mtime + 1, usegmt=True) + + assert get(**{'If-Modified-Since': before})['status'] == 200 + assert get(**{'If-Modified-Since': after})['status'] == 304 + assert get(**{'If-Unmodified-Since': before})['status'] == 412 + assert get(**{'If-Unmodified-Since': after})['status'] == 200 + + # Not an HTTP-date: ignored (Sect. 13.1.3 / 13.1.4), never a 412; as an + # If-Range it is a mismatch, so Range is ignored (200). + # (nxt_time_parse() is lenient about the zone token -- " GMT" missing, + # " XYZ", or trailing junk after a complete date all parse; not probed.) + for value in ['yesterday', '0', '1700000000', last_modified[:-8], '']: + assert get(**{'If-Modified-Since': value})['status'] == 200, value + assert get(**{'If-Unmodified-Since': value})['status'] == 200, value + r = get(Range='bytes=0-4', **{'If-Range': value}) + assert r['status'] == 200, f'If-Range {value!r} is a mismatch' + assert 'Content-Range' not in r['headers'] + + # If-Range wants an exact date; newer or older is a mismatch. + for value in [before, after]: + r = get(Range='bytes=0-4', **{'If-Range': value}) + assert r['status'] == 200, value + assert r['body'] == '0123456789' + + # A 304 carries the validators and no framing headers. + r = get(**{'If-Modified-Since': last_modified}) + assert r['status'] == 304 + assert r['headers']['Last-Modified'] == last_modified + assert 'ETag' in r['headers'] + assert 'Content-Length' not in r['headers'] + assert 'Transfer-Encoding' not in r['headers'] + + +def test_static_conditional_precedence(): + r = client.get(url='/index.html') + etag = r['headers']['ETag'] + last_modified = r['headers']['Last-Modified'] + old = 'Thu, 01 Jan 2000 00:00:00 GMT' + future = 'Fri, 01 Jan 2100 00:00:00 GMT' + + def get(**headers): + return client.get( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + # Sect. 13.2.2 step 1 -> 2: If-Unmodified-Since is ignored when If-Match + # is present, whichever way If-Match goes. + assert get(**{'If-Match': etag, 'If-Unmodified-Since': old})['status'] == 200 + assert get(**{'If-Match': '"x"', 'If-Unmodified-Since': future})[ + 'status' + ] == 412 + + # Step 3 -> 4: If-Modified-Since is ignored when If-None-Match is present. + assert get(**{'If-None-Match': '"x"', 'If-Modified-Since': future})[ + 'status' + ] == 200 + assert get(**{'If-None-Match': etag, 'If-Modified-Since': old})[ + 'status' + ] == 304 + + # A failed step 1/2 wins over a would-be 304 from step 3/4. + assert get(**{'If-Match': '"x"', 'If-None-Match': etag})['status'] == 412 + assert get(**{'If-Unmodified-Since': old, 'If-None-Match': etag})[ + 'status' + ] == 412 + assert get(**{'If-Unmodified-Since': old, 'If-Modified-Since': future})[ + 'status' + ] == 412 + + # A passed step 1 followed by a matching step 3 is a 304. + assert get(**{'If-Match': etag, 'If-None-Match': etag})['status'] == 304 + assert get(**{'If-Match': '*', 'If-None-Match': '*'})['status'] == 304 + + # Step 5: If-Range is consulted only after 1-4 pass. + r = get(Range='bytes=0-4', **{'If-Range': etag, 'If-None-Match': etag}) + assert r['status'] == 304 + assert 'Content-Range' not in r['headers'] + + r = get(Range='bytes=0-4', **{'If-Range': etag, 'If-Match': '"x"'}) + assert r['status'] == 412 + assert 'Content-Range' not in r['headers'] + + r = get(Range='bytes=0-4', **{'If-Range': last_modified, + 'If-Unmodified-Since': old}) + assert r['status'] == 412 + + r = get(Range='bytes=0-4', **{'If-Range': last_modified, + 'If-Modified-Since': last_modified}) + assert r['status'] == 304 + + # Preconditions pass but If-Range fails: full 200, not 206 and not 416. + r = get(Range='bytes=50-', **{'If-Range': '"x"', 'If-Match': etag}) + assert r['status'] == 200 + assert r['body'] == '0123456789' + assert 'Content-Range' not in r['headers'] + + # Preconditions pass, If-Range matches, range unsatisfiable: 416. + r = get(Range='bytes=50-', **{'If-Range': etag, 'If-None-Match': '"x"'}) + assert r['status'] == 416 + assert r['headers']['Content-Range'] == 'bytes */10' + + # A weak If-Range tag never matches (strong comparison, Sect. 13.1.5). + r = get(Range='bytes=0-4', **{'If-Range': f'W/{etag}'}) + assert r['status'] == 200 + assert r['body'] == '0123456789' + + # If-Range that is a list of tags is not the grammar; the only sane + # readings are "mismatch" (200) or a self-consistent 206. + r = get(Range='bytes=0-4', **{'If-Range': f'"x", {etag}'}) + assert r['status'] in (200, 206) + if r['status'] == 206: + assert r['headers']['Content-Range'] == 'bytes 0-4/10' + + # Every precondition + Range on a 412: no Content-Range. + for headers in [{'If-Match': '"x"'}, {'If-Unmodified-Since': old}]: + r = get(Range='bytes=0-4', **headers) + assert r['status'] == 412, headers + assert 'Content-Range' not in r['headers'], headers + + # Every precondition + Range on a 304: no Content-Range, no body, and a + # 304 wins over what would otherwise be a 416. + for headers in [{'If-None-Match': etag}, + {'If-Modified-Since': last_modified}]: + r = get(Range='bytes=0-4', **headers) + assert r['status'] == 304, headers + assert 'Content-Range' not in r['headers'], headers + assert r['body'] == '' + r = get(Range='bytes=50-', **headers) + assert r['status'] == 304, f'304 wins over 416 {headers}' + assert 'Content-Range' not in r['headers'], headers + + +def test_static_range_head(): + def head(**headers): + return client.head( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + resp = head(Range='bytes=2-6') + assert resp['status'] == 206, 'HEAD honours Range' + assert resp['body'] == '', 'no body on HEAD' + assert resp['headers']['Content-Range'] == 'bytes 2-6/10' + assert resp['headers']['Content-Length'] == '5', 'partial length on HEAD' + assert resp['headers']['Accept-Ranges'] == 'bytes' + + resp = head(Range='bytes=10-') + assert resp['status'] == 416 + assert resp['headers']['Content-Range'] == 'bytes */10' + assert resp['body'] == '' + + etag = client.get(url='/index.html')['headers']['ETag'] + resp = head(Range='bytes=2-6', **{'If-None-Match': etag}) + assert resp['status'] == 304 + assert 'Content-Range' not in resp['headers'] + + resp = head(Range='bytes=2-6', **{'If-Match': '"x"'}) + assert resp['status'] == 412 + + +def test_static_range_post_ignored(): + # Range is defined for GET only (Sect. 14.2); static serves GET/HEAD and + # answers 405 for the rest regardless of any Range header. + resp = client.post( + url='/index.html', + headers={'Host': 'localhost', 'Connection': 'close', + 'Range': 'bytes=0-4'}, + ) + assert resp['status'] == 405 + assert 'Content-Range' not in resp['headers'] + + +def _pattern(size): + # Byte i is (i * 7 + i // 251) & 0xff: aperiodic over any 128 KiB + # window, so a slice that is offset by even one byte, or fetched from + # the wrong buffer, compares unequal. + return bytes(((i * 7) + (i // 251)) & 0xFF for i in range(size)) + + +def _range_bytes(url, first, last=None, suffix=None): + if suffix is not None: + value = f'bytes=-{suffix}' + elif last is None: + value = f'bytes={first}-' + else: + value = f'bytes={first}-{last}' + + resp = client.get( + url=url, + headers={'Host': 'localhost', 'Connection': 'close', 'Range': value}, + encoding='latin-1', + read_buffer_size=1024 * 1024, + ) + resp['body'] = resp['body'].encode('latin-1') + return resp + + +def test_static_range_spans_buffers(temp_dir): + # NXT_HTTP_STATIC_BUF_SIZE is 128 KiB and NXT_HTTP_STATIC_BUF_COUNT + # buffers are issued per body-handler round; a 300 KiB file makes a + # mid-buffer start cross two boundaries and a second read round. + buf = 128 * 1024 + size = 300 * 1024 + data = _pattern(size) + Path(f'{temp_dir}/assets/big.bin').write_bytes(data) + + cases = [ + (0, size - 1), # whole file as a range + (0, buf - 1), # exactly the first buffer + (0, buf), # one past the first buffer + (1, buf), # off by one at both ends + (buf - 1, buf), # straddles the first boundary + (buf, 2 * buf - 1), # exactly the second buffer + (buf + 1, 2 * buf + 1), # mid-buffer start, two crossings + (100_000, 200_000), # arbitrary mid-buffer slice + (2 * buf + 5, size - 1), # tail past the second boundary + (size - 1, size - 1), # last byte + (12345, None), # open-ended from mid-buffer + (2 * buf + 17, None), # open-ended from the last buffer + ] + + for first, last in cases: + resp = _range_bytes('/big.bin', first, last) + end = size - 1 if last is None else min(last, size - 1) + assert resp['status'] == 206, (first, last) + assert resp['headers']['Content-Range'] == f'bytes {first}-{end}/{size}' + assert resp['headers']['Content-Length'] == str(end - first + 1) + assert len(resp['body']) == end - first + 1, (first, last) + assert resp['body'] == data[first : end + 1], ( + f'wrong bytes for {first}-{last}' + ) + + for suffix in [1, buf, buf + 1, 2 * buf + 3, size, size + 1]: + resp = _range_bytes('/big.bin', None, suffix=suffix) + first = max(size - suffix, 0) + assert resp['status'] == 206, suffix + assert resp['headers']['Content-Range'] == ( + f'bytes {first}-{size - 1}/{size}' + ) + assert resp['body'] == data[first:], f'wrong bytes for -{suffix}' + + # And the plain GET still returns everything, so the comparison base is + # not itself suspect. + resp = client.get(url='/big.bin', encoding='latin-1', + read_buffer_size=1024 * 1024) + assert resp['status'] == 200 + assert resp['body'].encode('latin-1') == data + + +def test_static_range_keepalive_mix(temp_dir): + # One connection, many requests: 206 / 200 / 416 / 304 in a row recycle + # the same per-thread buffer descriptor, so a stale file_pos or file_end + # would show up as a shifted or truncated body. The 304 and 416 also + # prove their framing (no body, connection stays usable). + size = 200 * 1024 + data = _pattern(size) + Path(f'{temp_dir}/assets/big.bin').write_bytes(data) + etag = client.get(url='/big.bin')['headers']['ETag'] + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.connect(('127.0.0.1', 8080)) + sock.settimeout(5) + + def request(extra=''): + sock.sendall( + ( + f'GET /big.bin HTTP/1.1\r\nHost: localhost\r\n' + f'Connection: keep-alive\r\n{extra}\r\n' + ).encode() + ) + raw = b'' + while b'\r\n\r\n' not in raw: + raw += sock.recv(65536) + head, _, rest = raw.partition(b'\r\n\r\n') + lines = head.decode().split('\r\n') + status = int(lines[0].split(' ')[1]) + headers = dict(line.split(': ', 1) for line in lines[1:]) + length = int(headers.get('Content-Length', 0)) + while len(rest) < length: + rest += sock.recv(65536) + assert len(rest) == length, 'no bytes after the declared body' + return status, headers, rest + + try: + steps = [ + ('Range: bytes=150000-180000\r\n', 206, data[150000:180001], + f'bytes 150000-180000/{size}'), + ('', 200, data, None), + ('Range: bytes=-10\r\n', 206, data[-10:], + f'bytes {size - 10}-{size - 1}/{size}'), + (f'Range: bytes={size}-\r\n', 416, b'', f'bytes */{size}'), + ('Range: bytes=0-0\r\n', 206, data[:1], f'bytes 0-0/{size}'), + (f'If-None-Match: {etag}\r\n', 304, b'', None), + ('Range: bytes=131071-131072\r\n', 206, data[131071:131073], + f'bytes 131071-131072/{size}'), + (f'Range: bytes=5-\r\nIf-None-Match: {etag}\r\n', 304, b'', None), + ('', 200, data, None), + ('Range: bytes=1-\r\n', 206, data[1:], f'bytes 1-{size - 1}/{size}'), + ] + + for i in range(3): + for extra, want_status, want_body, want_range in steps: + status, headers, body = request(extra) + assert status == want_status, (i, extra) + assert body == want_body, (i, extra, len(body)) + if want_range is None: + assert 'Content-Range' not in headers, (i, extra) + else: + assert headers['Content-Range'] == want_range, (i, extra) + finally: + sock.close() + + +def test_static_range_concurrent(temp_dir): + # Many clients pulling different slices of one file at the same time, + # each connection interleaving its reads with the others so the + # per-thread descriptor freelist is churned while bodies are in flight. + # A leaked file_pos would hand one client another client's bytes. + import threading + + size = 320 * 1024 + data = _pattern(size) + Path(f'{temp_dir}/assets/big.bin').write_bytes(data) + + ranges = [ + (0, 65535), (65536, 131071), (131072, 196607), (196608, 262143), + (262144, size - 1), (1, 300000), (131071, 131073), (100, 100), + (200000, None), (0, None), (size - 1, None), (77777, 233333), + ] + + errors = [] + barrier = threading.Barrier(len(ranges) * 2) + + def worker(first, last): + try: + for _ in range(4): + barrier.wait(timeout=10) + resp = _range_bytes('/big.bin', first, last) + end = size - 1 if last is None else last + if resp['status'] != 206: + errors.append((first, last, resp['status'])) + continue + if resp['headers']['Content-Range'] != ( + f'bytes {first}-{end}/{size}' + ): + errors.append((first, last, resp['headers']['Content-Range'])) + if resp['body'] != data[first : end + 1]: + got = resp['body'] + off = next( + (k for k in range(min(len(got), end - first + 1)) + if got[k] != data[first + k]), + min(len(got), end - first + 1), + ) + errors.append((first, last, 'body', len(got), off)) + except Exception as e: # pylint: disable=broad-except + errors.append((first, last, repr(e))) + + threads = [threading.Thread(target=worker, args=r) for r in ranges] + [ + threading.Thread(target=worker, args=r) for r in ranges + ] + for t in threads: + t.start() + for t in threads: + t.join(timeout=60) + + assert not errors, errors + + +def test_static_range_share_features(temp_dir): + assets = f'{temp_dir}/assets' + etag = client.get(url='/index.html')['headers']['ETag'] + + def get(url, **headers): + return client.get( + url=url, + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + # Index: "/" resolves to index.html; Range and validators apply to it. + r = get('/', Range='bytes=2-4') + assert r['status'] == 206, 'range on an index' + assert r['body'] == '234' + assert r['headers']['Content-Range'] == 'bytes 2-4/10' + assert get('/', **{'If-None-Match': etag})['status'] == 304 + assert get('/', **{'If-Match': '"x"'})['status'] == 412 + + # Directory redirect: a 301 is neither conditional nor partial. + for headers in [{'Range': 'bytes=0-1'}, {'If-None-Match': '*'}, + {'If-Match': '"x"'}, {'If-None-Match': etag}, + {'If-Modified-Since': 'Fri, 01 Jan 2100 00:00:00 GMT'}, + {'If-Unmodified-Since': 'Thu, 01 Jan 2000 00:00:00 GMT'}]: + r = get('/dir', **headers) + assert r['status'] == 301, headers + assert r['headers']['Location'] == '/dir/', headers + assert 'Content-Range' not in r['headers'], headers + assert 'Accept-Ranges' not in r['headers'], headers + + # 404 is not conditional either (Sect. 13.2.1: preconditions apply only + # when the unconditional response would be 2xx or 412). + for headers in [{'Range': 'bytes=0-1'}, {'If-None-Match': '*'}, + {'If-Match': '"x"'}, {'If-Match': '*'}]: + r = get('/nope', **headers) + assert r['status'] == 404, headers + assert 'Content-Range' not in r['headers'], headers + + # types: index.html is text/html; a share restricted to text/plain + # refuses it (403, as test_static_types.py documents), so Range and + # validators never get evaluated and the outcome is the same as without. + assert 'success' in client.conf( + {"share": f'{assets}$uri', "types": ["text/plain"]}, + 'routes/0/action', + ) + for headers in [{}, {'Range': 'bytes=0-1'}, {'If-None-Match': etag}, + {'If-Match': '"x"'}]: + r = get('/index.html', **headers) + assert r['status'] == 403, headers + assert 'Content-Range' not in r['headers'], headers + assert 'Accept-Ranges' not in r['headers'], headers + + r = get('/README', Range='bytes=0-2') + assert r['status'] == 206, 'type-matched file still supports ranges' + assert r['body'] == 'rea' + assert r['headers']['Content-Range'] == 'bytes 0-2/6' + assert r['headers']['Content-Type'] == 'text/plain' + + # fallback: an unreadable share falls through and the fallback answers, + # whatever the conditional/range headers were. + assert 'success' in client.conf( + { + "share": f'{assets}/missing$uri', + "fallback": {"return": 200}, + }, + 'routes/0/action', + ) + for headers in [{'Range': 'bytes=0-1'}, {'If-None-Match': '*'}, + {'If-Match': '"x"'}]: + r = get('/index.html', **headers) + assert r['status'] == 200, headers + assert 'Content-Range' not in r['headers'], headers + + # fallback to a second share: the first share's 404 carries no + # partially-built response into the second share's 206. + assert 'success' in client.conf( + { + "share": [f'{assets}/missing$uri', f'{assets}$uri'], + }, + 'routes/0/action', + ) + r = get('/index.html', Range='bytes=3-5') + assert r['status'] == 206 + assert r['body'] == '345' + assert r['headers']['Content-Range'] == 'bytes 3-5/10' + assert get('/index.html', **{'If-None-Match': etag})['status'] == 304 + + +def test_static_range_file_changes_between_requests(temp_dir): + # The ETag encodes mtime and size, so If-Range must stop applying once + # the file is rewritten (Sect. 13.1.5 exists for resumed downloads). + path = Path(f'{temp_dir}/assets/index.html') + r = client.get(url='/index.html') + etag = r['headers']['ETag'] + last_modified = r['headers']['Last-Modified'] + + path.write_text('abcdefghijklmnop', encoding='utf-8') + new_mtime = int(time.time()) + 5 + os.utime(path, (new_mtime, new_mtime)) + + r = range_get(Range='bytes=10-', **{'If-Range': etag}) + assert r['status'] == 200, 'stale If-Range tag ignores Range' + assert r['body'] == 'abcdefghijklmnop' + + r = range_get(Range='bytes=10-', **{'If-Range': last_modified}) + assert r['status'] == 200, 'stale If-Range date ignores Range' + + r = range_get(**{'If-None-Match': etag}) + assert r['status'] == 200, 'stale INM is a miss' + + r = range_get(**{'If-Match': etag}) + assert r['status'] == 412, 'stale If-Match fails' + + new_etag = client.get(url='/index.html')['headers']['ETag'] + r = range_get(Range='bytes=10-', **{'If-Range': new_etag}) + assert r['status'] == 206 + assert r['body'] == 'klmnop' + assert r['headers']['Content-Range'] == 'bytes 10-15/16' + + +def test_static_range_gzip(temp_dir): + # Compression must be skipped for a 206 (a coded slice would be + # meaningless) but still work for the plain 200 alongside it. The + # module keeps global state, so this needs a fresh unitd (#167). + import gzip + + from unit.option import option + + if not option.restart: + pytest.skip('needs --restart until #167 is fixed') + + resp = client.conf( + { + "http": { + "compression": { + "types": ["text/html*"], + "compressors": [{"encoding": "gzip", "level": 1}], + } + } + }, + 'settings', + ) + if 'supported compressor' in resp.get('detail', ''): + pytest.skip('unit built without gzip support') + assert 'success' in resp, resp + + size = 200 * 1024 + data = ('0123456789abcdef' * (size // 16)).encode() + Path(f'{temp_dir}/assets/big.html').write_bytes(data) + + def get(**headers): + raw = client.get( + url='/big.html', + headers={'Host': 'localhost', 'Connection': 'close', + 'Accept-Encoding': 'gzip', **headers}, + encoding='latin-1', + read_buffer_size=1024 * 1024, + raw_resp=True, + ) + head, _, body = raw.partition('\r\n\r\n') + lines = head.split('\r\n') + status = int(lines[0].split(' ')[1]) + headers = dict(line.split(': ', 1) for line in lines[1:]) + body = body.encode('latin-1') + if headers.get('Transfer-Encoding') == 'chunked': + body = client._parse_chunked_body(body) + return status, headers, body + + status, headers, body = get() + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip', 'plain GET compresses' + assert gzip.decompress(body) == data + + status, headers, body = get(Range='bytes=100000-200000') + assert status == 206 + assert 'Content-Encoding' not in headers, 'no coding on a 206' + assert headers['Content-Range'] == f'bytes 100000-200000/{size}' + assert headers['Content-Length'] == '100001' + assert body == data[100000:200001] + + status, headers, body = get(Range=f'bytes={size}-') + assert status == 416 + assert 'Content-Encoding' not in headers + assert headers['Content-Range'] == f'bytes */{size}' + + # A 200 after the 206 still compresses: the skip is per request. + status, headers, body = get() + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip' + + +def test_static_range_numeric_overflow(): + # Kept last in the file: the sign-flipped inputs below hang their + # connection and leak an open descriptor in the router, which the + # harness's teardown fd check reports and which then breaks every test + # that would run after it in the same unitd. + # RFC 9110 Sect. 14.1.2: a first-pos greater than or equal to the current + # length is unsatisfiable (416), whatever its magnitude. A value that + # does not fit nxt_off_t must not wrap into a "valid" offset. + # + # 2**63 flips the sign of a 64-bit signed accumulator, 2**64-1 wraps to + # -1, 2**64 wraps to 0. Any of these as the first-pos is >= size, so + # 416 (or 200) is right; a 206 for them is a wrap. + firsts = ['9223372036854775807', '9223372036854775808', + '9223372036854775809', '18446744073709551615', + '18446744073709551616', '18446744073709551617', + '99999999999999999999', '9' * 40, '9' * 4000] + + for first in firsts: + for value in [f'bytes={first}-', f'bytes={first}-{first}']: + resp = range_get(Range=value) + # Regression: a wrapped negative first-pos used to pass the + # "a >= size" test and be served as a 206 with a negative + # Content-Range start, leaking the open file descriptor. + assert resp['status'] != 206, ( + f'{value[:60]!r}: 206 {resp["headers"].get("Content-Range")}' + ) + _check_range_honest(value, resp) + + # The same magnitudes as last-pos: satisfiable, must clamp to size-1. + for last in firsts: + for first in ['0', '5']: + value = f'bytes={first}-{last}' + _check_range_honest(value, range_get(Range=value)) + + # And as a suffix-length (Sect. 14.1.2: longer than the representation + # means the whole representation). + for suffix in firsts: + value = f'bytes=-{suffix}' + _check_range_honest(value, range_get(Range=value)) + + # Negative-looking inputs are simply not the grammar. + for value in ['bytes=-5-9', 'bytes=-1-', 'bytes=5--9', 'bytes=-0-']: + _check_range_honest(value, range_get(Range=value)) From a0177579c85596781052416866d17c78b51a30c6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 23:38:58 +0200 Subject: [PATCH 160/391] unitctl: drop YAML input and output serde_yaml is archived since 2024-03-25 and pulls unsafe-libyaml, a c2rust transpile of libyaml with 241 unsafe sites, archived since 2024-03-21. RustSec has no unmaintained advisory for either (advisory-db#2132 is still open), so cargo audit does not warn and will keep not warning. Input and output cannot be separated. serde_yaml has no Cargo features, so linking it for one half links both, and the emitter is the transpiled code too: serde_yaml/src/libyaml/emitter.rs drives unsafe-libyaml's emitter.rs, 52 of those 241 unsafe sites. Every serde-integrated YAML crate on crates.io still carries unsafe-libyaml or its rename unsafe-libyaml-norway, so a swap keeps the transpile; serde_yaml_bw adds regex, base64, saphyr-parser, zmij and num-traits on top of it. A hand-written emitter would have to reproduce YAML 1.1 plain-scalar resolution over arbitrary operator strings, where a missing quote is silent data corruption. So both go, and yq covers the format. The Yaml input variant stays in InputFormat. Deleting it would make a ".yaml" file InputFormat::Unknown, and execute.rs and import.rs end their format branch with panic!("Unknown input file type"). Keeping it inside is_config() routes the file to a message naming YAML instead of an abort. "unitctl status -t yaml" is gone; clap refuses the value and lists the formats that remain. The docs site shows that example and needs the same change. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/Cargo.lock | 20 --------- tools/unitctl/README.md | 14 ++----- tools/unitctl/unitctl/Cargo.toml | 1 - tools/unitctl/unitctl/src/inputfile.rs | 48 +++++++++++++++------- tools/unitctl/unitctl/src/output_format.rs | 3 -- tools/unitctl/unitctl/src/unitctl.rs | 2 +- 6 files changed, 38 insertions(+), 50 deletions(-) diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 5512edf9c..a3116c9e3 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -1220,19 +1220,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "shlex" version = "2.0.1" @@ -1519,7 +1506,6 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "serde_yaml", "tar", "tempfile", "tokio", @@ -1528,12 +1514,6 @@ dependencies = [ "which", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/tools/unitctl/README.md b/tools/unitctl/README.md index bfb40bf70..c0c6379fe 100644 --- a/tools/unitctl/README.md +++ b/tools/unitctl/README.md @@ -80,8 +80,7 @@ Options: Print version ``` -- Consumes alternative configuration formats Like YAML and converts them -- Can convert output to multiple different formats (YAML, plain JSON, highlighted JSON) +- Can convert output to multiple different formats (plain JSON, highlighted JSON) - Syntactic highlighting of JSON output - Interpretation of Unit errors with (arguably more) useful error messages @@ -203,16 +202,9 @@ Unitctl can query the control API to provide the status of the running Unit daemon. ``` -$ unitctl status -t yaml +$ unitctl status -t json No socket path provided - attempting to detect from running instance -connections: - accepted: 0 - active: 0 - idle: 0 - closed: 0 -requests: - total: 0 -applications: {} +{"connections":{"accepted":0,"active":0,"idle":0,"closed":0},"requests":{"total":0},"applications":{}} ``` *Note:* This command supports operating on multiple instances of Unit at once. diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index eab68d1c2..6cd17bdda 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -18,7 +18,6 @@ custom_error = "1.9" serde = "1.0" json5 = "1.3.1" serde_json = { version = "1.0", optional = false } -serde_yaml = "0.9" # The "std" feature is not on by default and pem::from_buf needs it. rustls-pki-types = { version = "1.15", features = ["std"] } unit-client-rs = { path = "../unit-client-rs" } diff --git a/tools/unitctl/unitctl/src/inputfile.rs b/tools/unitctl/unitctl/src/inputfile.rs index d0fac0023..2860d2f7f 100644 --- a/tools/unitctl/unitctl/src/inputfile.rs +++ b/tools/unitctl/unitctl/src/inputfile.rs @@ -11,6 +11,9 @@ use super::UnitctlError; /// Input file data format #[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)] pub enum InputFormat { + /// Recognised, but not parsed. unitctl no longer reads YAML. The format + /// stays in this list so that a ".yaml" file gets a message saying that, + /// instead of a JSON parse error. Yaml, Json, Json5, @@ -144,8 +147,6 @@ impl InputFile { pub fn to_unit_serializable_map(&self) -> Result { let reader: Box = self.try_into()?; let body_data: UnitSerializableMap = match self.format() { - InputFormat::Yaml => serde_yaml::from_reader(reader) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?, InputFormat::Json => serde_json::from_reader(reader) .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?, InputFormat::Json5 => { @@ -157,12 +158,17 @@ impl InputFile { json5::from_str(&json5_string) .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })? } - // Refuse hjson by name. Handing the file to the JSON parser would - // report a syntax error on the first comment or unquoted key, and - // that error does not tell the user what to do. + // Refuse hjson and YAML by name. Handing the file to the JSON + // parser would report a syntax error on the first comment or + // unquoted key, and that error does not tell the user what to do. InputFormat::Hjson => Err(UnitctlError::DeserializationError { message: "hjson is no longer supported: convert the file to JSON first".to_string(), })?, + InputFormat::Yaml => Err(UnitctlError::DeserializationError { + message: "YAML is no longer supported: convert the file to JSON first, for example with \ + \"yq -o=json\"" + .to_string(), + })?, _ => Err(UnitctlError::DeserializationError { message: format!("Unsupported input format for serialization: {:?}", self), })?, @@ -311,9 +317,8 @@ mod tests { } /// `UnitSerializableMap` is a `serde_json::Map`, which keeps member order - /// only because every format feeding it keeps it too. `serde_yaml` and - /// `json5` each reach the type by a different route, so each one is - /// checked. + /// only because every format feeding it keeps it too. `json5` reaches the + /// type by a different route than `serde_json`, so both are checked. #[test] fn every_input_format_keeps_member_order() { let json = MEMBERS @@ -325,12 +330,6 @@ mod tests { assert_eq!(members_of(&write_input(&json, ".json"), InputFormat::Json), MEMBERS); assert_eq!(members_of(&write_input(&json, ".json5"), InputFormat::Json5), MEMBERS); - - let yaml = MEMBERS - .iter() - .map(|name| format!("{}: {{}}\n", name)) - .collect::(); - assert_eq!(members_of(&write_input(&yaml, ".yaml"), InputFormat::Yaml), MEMBERS); } /// hjson is refused by name. Handing the file to the JSON parser instead @@ -350,6 +349,27 @@ mod tests { ); } + /// YAML is refused by name for the same reason, and the ".yaml" extension + /// still has to reach that refusal. Were the variant dropped instead, the + /// file would become `InputFormat::Unknown` and `execute` would abort on + /// its `panic!("Unknown input file type")`. + #[test] + fn a_yaml_input_is_refused_with_a_message_about_yaml() { + for suffix in [".yaml", ".yml"] { + let file = write_input("routes: []\n", suffix); + let input = InputFile::from(file.path()); + + assert!(input.is_config(), "{} must stay a config format", suffix); + + let error = input.to_unit_serializable_map().expect_err("YAML must be refused"); + assert!( + error.to_string().contains("YAML is no longer supported"), + "unexpected message: {}", + error + ); + } + } + /// The way back in is strict, which is why `unitctl export` warns that the /// verbatim backup it writes for a configuration holding bytes that are not /// valid UTF-8 is one `unitctl import` cannot replay. diff --git a/tools/unitctl/unitctl/src/output_format.rs b/tools/unitctl/unitctl/src/output_format.rs index eb7f954e2..d5abbe226 100644 --- a/tools/unitctl/unitctl/src/output_format.rs +++ b/tools/unitctl/unitctl/src/output_format.rs @@ -6,7 +6,6 @@ use std::io::{stdout, BufWriter, Write}; #[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum OutputFormat { - Yaml, Json, #[value(id = "json-pretty")] JsonPretty, @@ -24,8 +23,6 @@ impl OutputFormat { serde_json::to_value(object).map_err(|e| UnitctlError::SerializationError { message: e.to_string() })?; match (self, no_color) { - (OutputFormat::Yaml, _) => serde_yaml::to_writer(BufWriter::new(out), &value) - .map_err(|e| UnitctlError::SerializationError { message: e.to_string() }), (OutputFormat::Json, _) => serde_json::to_writer(BufWriter::new(out), &value) .map_err(|e| UnitctlError::SerializationError { message: e.to_string() }), (OutputFormat::JsonPretty, true) => serde_json::to_writer_pretty(BufWriter::new(out), &value) diff --git a/tools/unitctl/unitctl/src/unitctl.rs b/tools/unitctl/unitctl/src/unitctl.rs index cb5f43da0..37b4e8039 100644 --- a/tools/unitctl/unitctl/src/unitctl.rs +++ b/tools/unitctl/unitctl/src/unitctl.rs @@ -84,7 +84,7 @@ pub(crate) enum Commands { global = true, short = 'f', long = "file", - help = "Input file (json, json5, yaml, pem) to send to unit when applicable use - for stdin" + help = "Input file (json, json5, pem) to send to unit when applicable use - for stdin" )] input_file: Option, From 5ae326c4100f9c775365528c2a32564f9df8da2f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 00:34:45 +0200 Subject: [PATCH 161/391] docs(changes): unitctl no longer reads or writes YAML 023cf7e8 (chore/unitctl-drop-serde-yaml) drops serde_yaml and unsafe-libyaml from unitctl. Merge this into the existing hjson removal entry rather than adding a second one: both are the same unreleased 1.36.2 stanza, and the hjson entry's "supported input formats are now json, json5, yaml and pem" sentence would otherwise sit right above a new entry contradicting it. Document the refusal messages for both ".hjson"/".cjson" and ".yaml"/".yml", the removed "unitctl status -t yaml" example, the narrowed --output-format values, and that tools/unitc keeps a supported YAML path via yq in both directions. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 27 +++++++++++++++++---------- docs/changes.xml | 23 +++++++++++++++-------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/CHANGES b/CHANGES index aa6cdf594..ff4bccd70 100644 --- a/CHANGES +++ b/CHANGES @@ -168,16 +168,23 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 must now speak HTTP/1.1 to unitctl; one that only accepts HTTP/2 can no longer be reached. - *) Change: unitctl no longer reads hjson. Input piped to "/config" on - stdin is parsed as JSON now, not hjson; hjson is a superset of JSON, - so piping JSON still works, but comments and unquoted keys do not. A - ".hjson" or ".cjson" file is refused with "hjson is no longer - supported: convert the file to JSON first", instead of being handed - to the JSON parser, which reports a misleading error at the first - comment. Supported input formats are now json, json5, yaml and - pem. This removes the nu-json dependency and the 30 crates of - Nushell utility code it pulled in for a tool that speaks one JSON - control API. + *) Change: unitctl no longer reads hjson or YAML, and no longer writes + YAML. Input piped to "/config" on stdin is parsed as JSON now, not + hjson; hjson is a superset of JSON, so piping JSON still works, but + comments and unquoted keys do not. A ".hjson" or ".cjson" file is + refused with "hjson is no longer supported: convert the file to JSON + first", instead of being handed to the JSON parser, which reports a + misleading error at the first comment. A ".yaml" or ".yml" file is + refused with "YAML is no longer supported: convert the file to JSON + first, for example with "yq -o=json"". Supported input formats are + now json, json5 and pem. "unitctl status -t yaml" is gone; + --output-format now accepts json, json-pretty and text. This removes + the nu-json dependency and the 30 crates of Nushell utility code it + pulled in for a tool that speaks one JSON control API. It also + removes serde_yaml and unsafe-libyaml; the latter is a c2rust + transpile of libyaml with 241 unsafe lines and no RustSec advisory. + tools/unitc is unaffected: it still converts configuration to and + from YAML, using yq. Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index 73b2f4e49..db47143be 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -224,14 +224,21 @@ HTTP/1.1 to unitctl; one that only accepts HTTP/2 can no longer be reached. -unitctl no longer reads hjson. Input piped to "/config" on stdin is parsed as -JSON now, not hjson; hjson is a superset of JSON, so piping JSON still works, -but comments and unquoted keys do not. A ".hjson" or ".cjson" file is refused -with "hjson is no longer supported: convert the file to JSON first", instead of -being handed to the JSON parser, which reports a misleading error at the first -comment. Supported input formats are now json, json5, yaml and pem. This -removes the nu-json dependency and the 30 crates of Nushell utility code it -pulled in for a tool that speaks one JSON control API. +unitctl no longer reads hjson or YAML, and no longer writes YAML. Input piped +to "/config" on stdin is parsed as JSON now, not hjson; hjson is a superset of +JSON, so piping JSON still works, but comments and unquoted keys do not. A +".hjson" or ".cjson" file is refused with "hjson is no longer supported: +convert the file to JSON first", instead of being handed to the JSON parser, +which reports a misleading error at the first comment. A ".yaml" or ".yml" +file is refused with "YAML is no longer supported: convert the file to JSON +first, for example with "yq -o=json"". Supported input formats are now json, +json5 and pem. "unitctl status -t yaml" is gone; --output-format now accepts +json, json-pretty and text. This removes the nu-json dependency and the 30 +crates of Nushell utility code it pulled in for a tool that speaks one JSON +control API. It also removes serde_yaml and unsafe-libyaml; the latter is a +c2rust transpile of libyaml with 241 unsafe lines and no RustSec advisory. +tools/unitc is unaffected: it still converts configuration to and from YAML, +using yq. From 7f9755ec2474d527a42ebf7b1dce12fe24e9a964 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 00:46:31 +0200 Subject: [PATCH 162/391] unitctl: put TLS behind a Cargo feature unitctl talks to the control API over a UNIX socket or over TCP. The control API has no TLS listener of its own, so the HTTPS connector is there for one case: a TLS-terminating proxy in front of a remote control API. That is a real deployment, so the connector stays. It is now a build-time choice. The cost of always having it is a C toolchain. hyper-rustls pulls in rustls, rustls pulls in aws-lc-rs, and aws-lc-sys builds a C library with cmake. Off, the dependency graph loses 11 crates counted with --edges normal (142 -> 131) and 19 counted with --edges normal,build (150 -> 131), among them aws-lc-sys, cc and cmake. The musl artifact is the one that wants this: it is meant to be a static binary that builds anywhere, and with the feature off it builds on a host that has neither cmake nor a musl C compiler. hyper-rustls and the direct rustls dependency are gated together. That rustls line is not used in any code; it is there so that rustls gets its own default features. Measured with "cargo tree -e features": every rustls feature except one is turned on twice over, by hyper-rustls's defaults as well, but "prefer-post-quantum" comes from the direct dependency alone. Gating one without the other would drop it, and cargo would not say a word. With the feature on the feature tree is byte-identical to before, so the provider is unchanged: aws-lc-rs, no ring. With the feature off an "https://" control socket is refused where the address is parsed, naming the feature, rather than being handed to a plain HTTP connector that would fail later with an unrelated message. The TCP connector type moves behind a type alias, so RemoteClient keeps both of its variants in both builds and no match arm is left dangling. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/GNUmakefile | 9 +++- tools/unitctl/unit-client-rs/Cargo.toml | 17 ++++++- .../src/control_socket_address.rs | 48 ++++++++++++++++++- .../unitctl/unit-client-rs/src/unit_client.rs | 42 +++++++++++----- tools/unitctl/unitctl/Cargo.toml | 7 ++- 5 files changed, 104 insertions(+), 19 deletions(-) diff --git a/tools/unitctl/GNUmakefile b/tools/unitctl/GNUmakefile index 975f50dfe..643ca0a8f 100644 --- a/tools/unitctl/GNUmakefile +++ b/tools/unitctl/GNUmakefile @@ -44,6 +44,11 @@ endif RELEASE_BUILD_FLAGS ?= --quiet --release --bin $(OUTPUT_BINARY) +# The musl artifact is built without the "tls" feature. TLS pulls in +# aws-lc-sys, a C library built by cmake, and that is the only thing making +# this build need a C toolchain. +target_feature_flags = $(if $(findstring musl,$(1)),--no-default-features,) + Q = $(if $(filter 1,$V),,@) M = $(shell printf "\033[34;1m▶\033[0m") @@ -68,8 +73,8 @@ all: $(TARGETS) ## Build all available platform targets [see: list-targets] .ONESHELL: $(TARGETS) $(TARGETS): openapi-generate $Q if [ ! -f "$(CURDIR)/target/$(@)/release/$(OUTPUT_BINARY)" ]; then - echo "$(M) building $(OUTPUT_BINARY) with flags [$(RELEASE_BUILD_FLAGS) --target $(@)]" - $(CARGO) build $(RELEASE_BUILD_FLAGS) --target $@ + echo "$(M) building $(OUTPUT_BINARY) with flags [$(RELEASE_BUILD_FLAGS) $(call target_feature_flags,$@) --target $(@)]" + $(CARGO) build $(RELEASE_BUILD_FLAGS) $(call target_feature_flags,$@) --target $@ fi target target/debug: diff --git a/tools/unitctl/unit-client-rs/Cargo.toml b/tools/unitctl/unit-client-rs/Cargo.toml index 2afc6cca2..bf34270ff 100644 --- a/tools/unitctl/unit-client-rs/Cargo.toml +++ b/tools/unitctl/unit-client-rs/Cargo.toml @@ -8,10 +8,20 @@ license = "Apache-2.0" [lib] name = "unit_client_rs" +[features] +default = ["tls"] +# "https://" control sockets. The control API has no TLS listener of its own, +# so this is for a TLS-terminating proxy in front of a remote control API. +# Turning it off drops aws-lc-sys, and with it the C toolchain the build needs; +# that is what the musl artifact does. hyper-rustls and rustls must be gated +# together: the direct rustls dependency is the only thing that turns on +# rustls's own default features, and losing one of those is silent. +tls = ["dep:hyper-rustls", "dep:rustls"] + [dependencies] custom_error = "1.9" hyper = { version = "1", features = ["http1"] } -hyper-rustls = { version = "0.27", features = ["rustls-native-certs", "webpki-roots"] } +hyper-rustls = { version = "0.27", features = ["rustls-native-certs", "webpki-roots"], optional = true } hyper-util = { version = "0.1", features = ["client-legacy", "client", "http1", "tokio"] } hyperlocal = "0.9" http-body-util = "0.1" @@ -28,7 +38,10 @@ hex = "0.4" which = "8.0" unit-openapi = { path = "../unit-openapi" } -rustls = "0.23.5" +# Not used in code. It is here so that rustls gets its own default features, +# which hyper-rustls does not turn on for us: "prefer-post-quantum" comes from +# this line alone. +rustls = { version = "0.23.5", optional = true } bollard = "0.21" regex = "1.10.4" pbr = "1.1.1" diff --git a/tools/unitctl/unit-client-rs/src/control_socket_address.rs b/tools/unitctl/unit-client-rs/src/control_socket_address.rs index a1077a4e1..9fda6ac09 100644 --- a/tools/unitctl/unit-client-rs/src/control_socket_address.rs +++ b/tools/unitctl/unit-client-rs/src/control_socket_address.rs @@ -106,7 +106,7 @@ impl TryFrom for ControlSocket { let path = String::from_utf8_lossy(&bytes); ControlSocket::parse_address(path) } - Some("http") | Some("https") => Ok(TcpSocket(socket_uri)), + Some("http") | Some("https") => ControlSocket::tcp_socket(socket_uri), Some(unknown) => Err(UnitClientError::TcpSocketAddressParseError { message: format!("Unsupported scheme found in socket address: {}", unknown).to_string(), control_socket_address: socket_uri.to_string(), @@ -120,6 +120,26 @@ impl TryFrom for ControlSocket { } impl ControlSocket { + /// Builds a TCP control socket from an already parsed URI. + #[cfg(feature = "tls")] + fn tcp_socket(uri: Uri) -> Result { + Ok(TcpSocket(uri)) + } + + /// Without the "tls" feature there is no HTTPS connector, so refuse the + /// address here. A plain HTTP connector would otherwise be handed an + /// "https://" URI and fail later with an unrelated message. + #[cfg(not(feature = "tls"))] + fn tcp_socket(uri: Uri) -> Result { + if uri.scheme_str() == Some("https") { + return Err(UnitClientError::TcpSocketAddressParseError { + message: "https needs the \"tls\" feature, which this build does not have".to_string(), + control_socket_address: uri.to_string(), + }); + } + Ok(TcpSocket(uri)) + } + pub fn socket_scheme(&self) -> ControlSocketScheme { match self { UnixLocalAbstractSocket(_) => ControlSocketScheme::HTTP, @@ -302,7 +322,7 @@ impl ControlSocket { } let uri = Self::normalize_and_parse_http_address(buf)?; - Ok(TcpSocket(uri)) + Self::tcp_socket(uri) } pub fn is_local_socket(&self) -> bool { @@ -458,6 +478,12 @@ mod tests { "[0000:0000:0000:0000:0000:0000:0000:0000]:8080", ]; for socket_address in valid_socket_addresses { + // A build without the "tls" feature refuses https; see + // refuses_https_without_the_tls_feature below. + if cfg!(not(feature = "tls")) && socket_address.starts_with("https://") { + continue; + } + let mut expected = if socket_address.starts_with("http") { socket_address.to_string().trim_end_matches('/').to_string() } else { @@ -506,6 +532,24 @@ mod tests { assert_eq!(normalized, expected); } + #[cfg(feature = "tls")] + #[test] + fn accepts_https_with_the_tls_feature() { + let control_socket = ControlSocket::try_from("https://localhost:8443").expect("https should be accepted"); + assert_eq!(control_socket.to_string(), "https://localhost:8443/"); + } + + #[cfg(not(feature = "tls"))] + #[test] + fn refuses_https_without_the_tls_feature() { + let error = ControlSocket::try_from("https://localhost:8443").expect_err("https should be refused"); + assert_eq!( + error.to_string(), + "Invalid TCP socket address [control_socket_address=https://localhost:8443/]: \ + https needs the \"tls\" feature, which this build does not have" + ); + } + #[test] fn can_parse_http_addresses() { let valid_socket_addresses = vec![ diff --git a/tools/unitctl/unit-client-rs/src/unit_client.rs b/tools/unitctl/unit-client-rs/src/unit_client.rs index 1b1a5c6e6..ecdcd7262 100644 --- a/tools/unitctl/unit-client-rs/src/unit_client.rs +++ b/tools/unitctl/unit-client-rs/src/unit_client.rs @@ -8,6 +8,7 @@ use bytes::Bytes; use custom_error::custom_error; use http_body_util::{BodyExt, Full}; use hyper::{http, Request}; +#[cfg(feature = "tls")] use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::client::legacy::{Client, ResponseFuture}; @@ -132,21 +133,29 @@ macro_rules! new_openapi_client { }; } +/// Connector for a TCP control socket. Without the "tls" feature there is no +/// HTTPS connector, and an "https://" address is refused while it is parsed. +#[cfg(feature = "tls")] +type TcpConnector = HttpsConnector; +#[cfg(not(feature = "tls"))] +type TcpConnector = HttpConnector; + +#[cfg(feature = "tls")] +const TCP_CLIENT_NAME: &str = "Client, Full>"; +#[cfg(not(feature = "tls"))] +const TCP_CLIENT_NAME: &str = "Client>"; + #[derive(Clone)] pub enum RemoteClient { - Unix { - client: Client>, - }, - Tcp { - client: Client, Full>, - }, + Unix { client: Client> }, + Tcp { client: Client> }, } impl RemoteClient { fn client_name(&self) -> &str { match self { RemoteClient::Unix { .. } => "Client>", - RemoteClient::Tcp { .. } => "Client, Full>", + RemoteClient::Tcp { .. } => TCP_CLIENT_NAME, } } @@ -236,17 +245,26 @@ impl UnitClient { } } - pub fn new_http(control_socket: ControlSocket) -> Self { - let connector = HttpsConnectorBuilder::new() + #[cfg(feature = "tls")] + fn tcp_connector() -> TcpConnector { + HttpsConnectorBuilder::new() .with_native_roots() .unwrap_or_else(|_| HttpsConnectorBuilder::new().with_webpki_roots()) .https_or_http() // The control API speaks HTTP/1.1 only. This also keeps "h2" out // of the ALPN list we offer. .enable_http1() - .build(); - let remote_client: Client, Full> = - Client::builder(TokioExecutor::new()).build(connector); + .build() + } + + #[cfg(not(feature = "tls"))] + fn tcp_connector() -> TcpConnector { + HttpConnector::new() + } + + pub fn new_http(control_socket: ControlSocket) -> Self { + let remote_client: Client> = + Client::builder(TokioExecutor::new()).build(Self::tcp_connector()); Self { control_socket, client: Box::from(RemoteClient::Tcp { client: remote_client }), diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index 6cd17bdda..48549b47c 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -11,6 +11,9 @@ name = "unitctl" path = "src/main.rs" [features] +default = ["tls"] +# "https://" control sockets. The musl build passes --no-default-features. +tls = ["unit-client-rs/tls"] [dependencies] clap = { version = "4.4", features = ["default", "derive", "cargo"] } @@ -20,7 +23,9 @@ json5 = "1.3.1" serde_json = { version = "1.0", optional = false } # The "std" feature is not on by default and pem::from_buf needs it. rustls-pki-types = { version = "1.15", features = ["std"] } -unit-client-rs = { path = "../unit-client-rs" } +# default-features = false, or unitctl's own --no-default-features would not +# reach unit-client-rs and TLS would stay on. +unit-client-rs = { path = "../unit-client-rs", default-features = false } colored_json = "5.0.0" tempfile = "3.8" which = "8.0.2" From 2083d844b32cd39a92f013915e41def124a03e2d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 01:12:51 +0200 Subject: [PATCH 163/391] docs(changes): record the unitctl tls build feature The musl build no longer accepts an https:// control socket address, and there is a new build option, so both belong in the release notes rather than only in the PR. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 6 ++++++ docs/changes.xml | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/CHANGES b/CHANGES index ff4bccd70..5c7d1d2f2 100644 --- a/CHANGES +++ b/CHANGES @@ -15,6 +15,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 evaluated first, so a 304 or 412 outcome wins over any Range. A response served from a Range is not content-coded; compression still applies to a full 200. + *) Change: unitctl builds TLS support behind a "tls" Cargo feature, on by + default. The musl build turns it off, so the static binary no longer + links a C TLS library and no longer accepts an "https://" control + socket address; it refuses one with a message naming the feature. A + Unix socket or a plain "http://" address is unaffected, and a build + with default features behaves as before. *) Feature: conditional requests for static files. A "share" action now answers a GET or HEAD carrying "If-None-Match" or "If-Modified-Since" diff --git a/docs/changes.xml b/docs/changes.xml index db47143be..1969944bb 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -242,6 +242,16 @@ using yq. + + +unitctl builds TLS support behind a "tls" Cargo feature, on by default. The +musl build turns it off, so the static binary no longer links a C TLS library +and no longer accepts an "https://" control socket address; it refuses one with +a message naming the feature. A Unix socket or a plain "http://" address is +unaffected, and a build with default features behaves as before. + + + From 7742b16fa9fd3430372e243f2de074931f9f638e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 02:05:38 +0200 Subject: [PATCH 164/391] build(unitctl): opt out of TLS on a musl host, not only a musl target target_feature_flags keys on a target name, so it reached only the cross builds that name one. A native build on a musl host passes no --target: "make release" on Alpine, which is how a distro packages this, kept TLS and compiled aws-lc-sys. That is the case the feature exists for. Ask rustc for the host triple and apply the same rule to the native debug and release recipes. HOST_TARGET is empty when rustc is absent, which leaves the flags unchanged. Verified with make -n: a glibc host builds unchanged; with the host reported as musl both native recipes gain --no-default-features; the cross rule is untouched. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/GNUmakefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/unitctl/GNUmakefile b/tools/unitctl/GNUmakefile index 643ca0a8f..ae51e4231 100644 --- a/tools/unitctl/GNUmakefile +++ b/tools/unitctl/GNUmakefile @@ -49,6 +49,14 @@ RELEASE_BUILD_FLAGS ?= --quiet --release --bin $(OUTPUT_BINARY) # this build need a C toolchain. target_feature_flags = $(if $(findstring musl,$(1)),--no-default-features,) +# The rule above keys on a target name, so it only reaches the cross builds that +# name one. A native build on a musl host -- "make release" on Alpine, which is +# how a distro packages this -- passes no --target and would otherwise keep TLS +# and compile aws-lc-sys. Ask rustc for the host triple and apply the same +# rule. Empty if rustc is absent, which leaves the flags unchanged. +HOST_TARGET := $(shell rustc -vV 2>/dev/null | sed -n 's/^host: //p') +HOST_FEATURE_FLAGS := $(call target_feature_flags,$(HOST_TARGET)) + Q = $(if $(filter 1,$V),,@) M = $(shell printf "\033[34;1m▶\033[0m") @@ -85,14 +93,14 @@ debug: target/debug/$(OUTPUT_BINARY) target/debug/$(OUTPUT_BINARY): openapi-generate $Q echo "$(M) building $(OUTPUT_BINARY) in debug mode for the current platform" - $Q $(CARGO) build --bin $(OUTPUT_BINARY) + $Q $(CARGO) build --bin $(OUTPUT_BINARY) $(HOST_FEATURE_FLAGS) .PHONY: release release: target/release/$(OUTPUT_BINARY) target/release/$(OUTPUT_BINARY): openapi-generate $Q echo "$(M) building $(OUTPUT_BINARY) in release mode for the current platform" - $Q $(CARGO) build $(RELEASE_BUILD_FLAGS) + $Q $(CARGO) build $(RELEASE_BUILD_FLAGS) $(HOST_FEATURE_FLAGS) .PHONY: test test: openapi-generate ## Run tests From cb781b8eb9d1cc939c2dc5d554ab3d1b960dca87 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 02:12:25 +0200 Subject: [PATCH 165/391] build(unitctl): opt the test recipe out of TLS on a musl host too The release and debug recipes take HOST_FEATURE_FLAGS; "make test" did not, so on a musl host it still built aws-lc-sys through cmake, and refuses_https_without_the_tls_feature never ran there -- the one test that covers the state this feature exists for. Also correct a number in the merged YAML entry. It said unsafe-libyaml carries 241 unsafe lines. On the pinned 0.2.11 that matches nothing: src/ has 224 lines containing "unsafe", 218 of them "unsafe fn", and 249 across the whole crate including tests. Say 218 unsafe functions, which is scoped and reproducible with grep -rho 'unsafe fn' src/*.rs | wc -l. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 3 ++- docs/changes.xml | 2 +- tools/unitctl/GNUmakefile | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 5c7d1d2f2..17784e921 100644 --- a/CHANGES +++ b/CHANGES @@ -188,7 +188,8 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 the nu-json dependency and the 30 crates of Nushell utility code it pulled in for a tool that speaks one JSON control API. It also removes serde_yaml and unsafe-libyaml; the latter is a c2rust - transpile of libyaml with 241 unsafe lines and no RustSec advisory. + transpile of libyaml with 218 unsafe functions and no RustSec + advisory. tools/unitc is unaffected: it still converts configuration to and from YAML, using yq. diff --git a/docs/changes.xml b/docs/changes.xml index 1969944bb..7977293cf 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -236,7 +236,7 @@ json5 and pem. "unitctl status -t yaml" is gone; --output-format now accepts json, json-pretty and text. This removes the nu-json dependency and the 30 crates of Nushell utility code it pulled in for a tool that speaks one JSON control API. It also removes serde_yaml and unsafe-libyaml; the latter is a -c2rust transpile of libyaml with 241 unsafe lines and no RustSec advisory. +c2rust transpile of libyaml with 218 unsafe functions and no RustSec advisory. tools/unitc is unaffected: it still converts configuration to and from YAML, using yq. diff --git a/tools/unitctl/GNUmakefile b/tools/unitctl/GNUmakefile index ae51e4231..dd1f2fe87 100644 --- a/tools/unitctl/GNUmakefile +++ b/tools/unitctl/GNUmakefile @@ -104,7 +104,7 @@ target/release/$(OUTPUT_BINARY): openapi-generate .PHONY: test test: openapi-generate ## Run tests - $Q $(CARGO) test + $Q $(CARGO) test $(HOST_FEATURE_FLAGS) .ONESHELL: target/man/$(OUTPUT_BINARY).1.gz target/man/$(OUTPUT_BINARY).1.gz: From 023d45c13fd6afcc5e5bc66dc20b57696d5f1180 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 00:58:36 +0200 Subject: [PATCH 166/391] http: say what a negotiated response varied on Unit emitted no Vary header, ever -- "grep -rn Vary src/" found nothing. With compression configured the same URL returns either an identity or a gzip representation depending on the request, so a shared cache was entitled to store whichever it saw first and serve it to everyone: gzip bytes to a client that cannot decode them, or the uncompressed copy to one that could have had the small version. Emit "Vary: Accept-Encoding" once the response is known to have been negotiated -- past every early return in nxt_http_comp_check_acceptable() that means "this was never subject to negotiation", so a file outside the configured types, or a response with compression disabled, does not carry it. A Vary already set by response_headers or another producer is left alone. Sent on the identity response as well as the coded one. The identity response is precisely the one a cache must not reuse for a gzip-capable client, so omitting it there would leave the hole open from the other side. This completes the pair started by weakening the entity-tag of a coded representation: that made revalidation distinguish the two representations, this makes the cache key distinguish them. Either alone still lets a shared cache mix them. Closes freeunitorg/freeunit#357. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- CHANGES | 9 +++++ docs/changes.xml | 12 ++++++ src/nxt_http_compression.c | 66 +++++++++++++++++++++++++++++++++ test/test_static_compression.py | 29 +++++++++++++++ 4 files changed, 116 insertions(+) diff --git a/CHANGES b/CHANGES index 17784e921..47fa8ee6d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: a static response that was subject to content negotiation now + carries "Vary: Accept-Encoding". Without it a shared cache could store + whichever representation it saw first and serve it to every client -- + gzip bytes to one that cannot decode them, or the uncompressed copy to + one that could have had the small version. The header is sent on the + identity response as well as the compressed one, since the identity + response is the one a cache must not reuse for a client that accepts + gzip. + *) Feature: byte-range requests for static files. A "share" action now answers a satisfiable single-range "Range: bytes=..." request with "206 Partial Content" and a matching "Content-Range", and an diff --git a/docs/changes.xml b/docs/changes.xml index 7977293cf..abfda99c4 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -36,6 +36,18 @@ representation, per RFC 9110. + + +a static response that was subject to content negotiation now carries +"Vary: Accept-Encoding". Without it a shared cache could store whichever +representation it saw first and serve it to every client -- gzip bytes to one +that cannot decode them, or the uncompressed copy to one that could have had +the small version. The header is sent on the identity response as well as +the compressed one, since the identity response is the one a cache must not +reuse for a client that accepts gzip. + + + a content-coded response now carries a weak entity-tag, so "If-Range" cannot diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index d2d659cfd..f44dac809 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -603,6 +603,63 @@ nxt_http_comp_is_resp_content_encoded(const nxt_http_request_t *r) } +/* + * Adds "Vary: Accept-Encoding", so a shared cache keys on the header that + * chose this representation. + * + * RFC 9110 Sect. 12.5.5: a response that was subject to proactive negotiation + * must say which request headers it varied on, or a cache is entitled to + * serve it to a client that would have been given a different representation + * -- gzip bytes to a client that cannot decode them, or identity to one that + * could have had the small copy. + * + * Emitted on the identity response as well as the coded one. The identity + * response is precisely the one a cache must not reuse for a gzip-capable + * client, so omitting it there would leave the hole open from the other side. + * + * This is the companion of weakening the entity-tag for a coded + * representation: that makes revalidation distinguish the two, this makes the + * cache key distinguish them. Either alone leaves shared caches able to mix + * them. + */ + +static nxt_int_t +nxt_http_comp_set_vary(nxt_http_request_t *r) +{ + nxt_http_field_t *f; + nxt_http_fields_iter_t iter; + + static const nxt_str_t accept_encoding = nxt_string("Accept-Encoding"); + + /* A Vary set by response_headers or another producer is left alone. */ + + for (f = nxt_http_fields_first(&iter, r->resp.inline_fields, + r->resp.num_inline_fields, r->resp.fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (!f->skip && f->name_length == nxt_length("Vary") + && nxt_strncasecmp(f->name, (u_char *) "Vary", + nxt_length("Vary")) == 0) + { + return NXT_OK; + } + } + + f = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(f == NULL)) { + return NXT_ERROR; + } + + nxt_http_field_name_set(f, "Vary"); + + f->value = accept_encoding.start; + f->value_length = accept_encoding.length; + + return NXT_OK; +} + + /* * Decides whether an acceptable representation exists, and remembers which * compressor would be used, without touching the response or allocating a @@ -665,6 +722,15 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_OK; } + /* + * Past every early return above, so this response really was subject to + * negotiation on Accept-Encoding, whichever coding is chosen below. + */ + + if (nxt_slow_path(nxt_http_comp_set_vary(r) != NXT_OK)) { + return NXT_ERROR; + } + ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, &r->tstr_cache, r, r->mem_pool); if (nxt_slow_path(ret == NXT_ERROR)) { diff --git a/test/test_static_compression.py b/test/test_static_compression.py index c8a1a43f1..9ec5b1f28 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -100,3 +100,32 @@ def test_static_compression_304_carries_no_encoding(): assert resp['body'] == '', 'no body' assert 'Content-Encoding' not in resp['headers'], 'no Content-Encoding' assert 'Content-Length' not in resp['headers'], 'no Content-Length' + + +def test_static_compression_vary(): + # RFC 9110 Sect. 12.5.5: a response subject to proactive negotiation must + # say what it varied on, or a shared cache may serve gzip bytes to a + # client that cannot decode them. This is the companion of the weak + # entity-tag: that makes revalidation distinguish the codings, this makes + # the cache key distinguish them. + def get(**headers): + return client.get( + url='/big.css', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + ) + + resp = get(**{'Accept-Encoding': 'gzip'}) + assert resp['status'] == 200 + assert resp['headers']['Content-Encoding'] == 'gzip', 'compressed' + assert ( + resp['headers']['Vary'] == 'Accept-Encoding' + ), 'Vary on the coded response' + + # The identity response is the one a cache must not reuse for a + # gzip-capable client, so it needs the header just as much. + resp = get() + assert resp['status'] == 200 + assert 'Content-Encoding' not in resp['headers'], 'not compressed' + assert ( + resp['headers']['Vary'] == 'Accept-Encoding' + ), 'Vary on the identity response too' From fd5b5373f0b9fd1d980c9d4b2ef163e57037b013 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 01:34:11 +0200 Subject: [PATCH 167/391] http: merge Accept-Encoding into an existing Vary The first version returned early when any Vary was already present, on the reasoning that another producer's header should be left alone. That is backwards: a response carrying "Vary: Origin" still varies on the coding too, and leaving it untouched keeps Accept-Encoding out of the cache key -- the exact hole this is meant to close, now with the header present and looking correct. Merge instead. An existing "Vary: *" is left alone, since it already varies on everything and narrowing it would say less. A list that already names Accept-Encoding is left alone. Otherwise the token is appended. Comparison is per token rather than by substring, so "X-Accept-Encoding" is correctly treated as a different header and gets Accept-Encoding appended. Reachable from an application response: the static path has nothing that sets Vary before this runs. Measured with a PHP app that sets the header and a Content-Length, gzip configured for text/html*: (none) -> Accept-Encoding Origin -> Origin, Accept-Encoding Accept-Encoding -> Accept-Encoding * -> * X-Accept-Encoding -> X-Accept-Encoding, Accept-Encoding A Vary set through "response_headers" still wins, because nxt_http_set_headers() runs after this and replaces the value outright. That is the same precedence "response_headers" has over every generated header, and matches how an ETag override is handled; it is a deliberate operator override rather than something to defend against here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- src/nxt_http_compression.c | 74 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index f44dac809..e88fe0b42 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -626,12 +626,22 @@ nxt_http_comp_is_resp_content_encoded(const nxt_http_request_t *r) static nxt_int_t nxt_http_comp_set_vary(nxt_http_request_t *r) { - nxt_http_field_t *f; + u_char *p, *end, *tok; + nxt_int_t len; + nxt_http_field_t *f, *vary; nxt_http_fields_iter_t iter; static const nxt_str_t accept_encoding = nxt_string("Accept-Encoding"); - /* A Vary set by response_headers or another producer is left alone. */ + /* + * An existing Vary is merged into, not replaced and not deferred to. + * Something else naming a different header -- "Vary: Origin", say -- + * still needs Accept-Encoding added, because the response varies on both; + * treating any existing Vary as sufficient would leave the coding out of + * the cache key, which is the hole this function exists to close. + */ + + vary = NULL; for (f = nxt_http_fields_first(&iter, r->resp.inline_fields, r->resp.num_inline_fields, r->resp.fields); @@ -642,8 +652,68 @@ nxt_http_comp_set_vary(nxt_http_request_t *r) && nxt_strncasecmp(f->name, (u_char *) "Vary", nxt_length("Vary")) == 0) { + vary = f; + break; + } + } + + if (vary != NULL) { + p = vary->value; + end = p + vary->value_length; + + /* "Vary: *" already varies on everything; adding to it says less. */ + + while (p < end && (*p == ' ' || *p == '\t')) { + p++; + } + + if (end - p == 1 && *p == '*') { return NXT_OK; } + + /* Already listed? Compare per token, so "X-Accept-Encoding" misses. */ + + for (p = vary->value; p < end; p++) { + while (p < end && (*p == ' ' || *p == '\t' || *p == ',')) { + p++; + } + + tok = p; + + while (p < end && *p != ',') { + p++; + } + + len = p - tok; + + while (len > 0 && (tok[len - 1] == ' ' || tok[len - 1] == '\t')) { + len--; + } + + if (len == (nxt_int_t) accept_encoding.length + && nxt_strncasecmp(tok, accept_encoding.start, + accept_encoding.length) == 0) + { + return NXT_OK; + } + } + + len = vary->value_length + nxt_length(", ") + accept_encoding.length; + + p = nxt_mp_nget(r->mem_pool, len); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + nxt_memcpy(p, vary->value, vary->value_length); + nxt_memcpy(p + vary->value_length, ", ", nxt_length(", ")); + nxt_memcpy(p + vary->value_length + nxt_length(", "), + accept_encoding.start, accept_encoding.length); + + vary->value = p; + vary->value_length = len; + + return NXT_OK; } f = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); From f647d88797291156bbd9f3e62e4c8aa61436e3d9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 01:47:11 +0200 Subject: [PATCH 168/391] http: trim both ends before treating Vary as a wildcard The wildcard test skipped leading whitespace but not trailing, so "Vary: * " fell through to the append and produced "* , Accept-Encoding" -- not a valid field value, and a narrowing of a header that already varies on everything. Trim both ends before the comparison, and re-anchor the token scan to the field's real end afterwards so the trim cannot shorten what is copied. These are response fields an application handed to libunit, not request headers the parser has normalised, so trailing whitespace can genuinely arrive here. Honest limit on the verification: PHP appears to normalise the trailing space before libunit sees it, so "Vary: * " through a PHP app reaches this code already trimmed and the new branch is not exercised by that test. The wildcard, leading-space and both-ends cases all return "*" unchanged, and "Origin " merges to "Origin, Accept-Encoding" with no stray space -- but a runtime that does not normalise is what the trim is there for, and I have not demonstrated one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- src/nxt_http_compression.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index e88fe0b42..f4635fd45 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -661,18 +661,32 @@ nxt_http_comp_set_vary(nxt_http_request_t *r) p = vary->value; end = p + vary->value_length; - /* "Vary: *" already varies on everything; adding to it says less. */ + /* + * "Vary: *" already varies on everything; adding to it says less. + * + * Trim both ends before the test. These are response fields an + * application handed to libunit, not request headers the parser has + * normalised, so "Vary: * " arrives with its trailing space intact -- + * and appending to that would emit "* , Accept-Encoding", which is + * not a valid field value. + */ while (p < end && (*p == ' ' || *p == '\t')) { p++; } + while (end > p && (end[-1] == ' ' || end[-1] == '\t')) { + end--; + } + if (end - p == 1 && *p == '*') { return NXT_OK; } /* Already listed? Compare per token, so "X-Accept-Encoding" misses. */ + end = vary->value + vary->value_length; + for (p = vary->value; p < end; p++) { while (p < end && (*p == ' ' || *p == '\t' || *p == ',')) { p++; From 2038b1b504d385ea0b903082ea196a8917587479 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 02:55:47 +0200 Subject: [PATCH 169/391] http: tidy the Vary token scan and its edges Two observations from review, both minor, both worth closing. The outer loop was a "for" with p++, and the inner scan can leave p at end; incrementing there forms a pointer past one-past-the-end, which C does not define even where every real target treats it as harmless. A while loop that advances only when there is something to advance past avoids constructing the pointer at all. An empty existing value made the append emit a leading comma -- ", Accept-Encoding". There are no tokens to search or append to in that case, so replace rather than append. The mirror case turned up while testing the first: a value ending in a separator, "Origin,", appended to give "Origin,, Accept-Encoding". Both forms are legal -- Sect. 5.6.1.2 permits empty list elements and readers ignore them -- but there is no reason to emit one. The append now starts after the last real token. Measured across the edges, gzip configured for text/html*: (empty) -> Accept-Encoding Origin -> Origin, Accept-Encoding a, b, c -> a, b, c, Accept-Encoding Origin, -> Origin, Accept-Encoding Origin, -> Origin, Accept-Encoding a, Accept-Encoding -> a, Accept-Encoding * -> * * -> * Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- src/nxt_http_compression.c | 57 ++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index f4635fd45..cab09e406 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -628,6 +628,7 @@ nxt_http_comp_set_vary(nxt_http_request_t *r) { u_char *p, *end, *tok; nxt_int_t len; + size_t keep; nxt_http_field_t *f, *vary; nxt_http_fields_iter_t iter; @@ -683,11 +684,34 @@ nxt_http_comp_set_vary(nxt_http_request_t *r) return NXT_OK; } - /* Already listed? Compare per token, so "X-Accept-Encoding" misses. */ + /* + * An empty value carries no tokens, so there is nothing to append to + * and nothing to search: replacing it avoids emitting ", A-E" with a + * leading comma. RFC 9110 Sect. 5.6.1.2 permits the empty element, + * but there is no reason to produce one. + */ end = vary->value + vary->value_length; - for (p = vary->value; p < end; p++) { + if (end == vary->value) { + vary->value = accept_encoding.start; + vary->value_length = accept_encoding.length; + + return NXT_OK; + } + + /* + * Already listed? Compare per token, so "X-Accept-Encoding" misses. + * + * A while loop rather than a for with p++: the inner scan can leave p + * at end, and incrementing there would form a pointer past + * one-past-the-end, which C does not define even where it is + * harmless in practice. + */ + + p = vary->value; + + while (p < end) { while (p < end && (*p == ' ' || *p == '\t' || *p == ',')) { p++; } @@ -710,18 +734,39 @@ nxt_http_comp_set_vary(nxt_http_request_t *r) { return NXT_OK; } + + if (p < end) { + p++; + } + } + + /* + * Append after the last real token, not after whatever the value + * happens to end with: "Origin," would otherwise become + * "Origin,, Accept-Encoding". Empty list elements are legal and + * ignored (Sect. 5.6.1.2), but there is no reason to emit one. + */ + + keep = vary->value_length; + + while (keep > 0 + && (vary->value[keep - 1] == ' ' + || vary->value[keep - 1] == '\t' + || vary->value[keep - 1] == ',')) + { + keep--; } - len = vary->value_length + nxt_length(", ") + accept_encoding.length; + len = keep + nxt_length(", ") + accept_encoding.length; p = nxt_mp_nget(r->mem_pool, len); if (nxt_slow_path(p == NULL)) { return NXT_ERROR; } - nxt_memcpy(p, vary->value, vary->value_length); - nxt_memcpy(p + vary->value_length, ", ", nxt_length(", ")); - nxt_memcpy(p + vary->value_length + nxt_length(", "), + nxt_memcpy(p, vary->value, keep); + nxt_memcpy(p + keep, ", ", nxt_length(", ")); + nxt_memcpy(p + keep + nxt_length(", "), accept_encoding.start, accept_encoding.length); vary->value = p; From e9acfe07097cbf94f576a97f05844f3a4e8a23b1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 03:02:19 +0200 Subject: [PATCH 170/391] http: keep Accept-Encoding in Vary past response_headers nxt_http_set_headers() runs after the Vary is generated and replaces a field it names outright, so "response_headers": {"Vary": "Origin"} emitted only "Vary: Origin", and {"Vary": null} removed the header entirely. Either way the response still varied on Accept-Encoding, so a shared cache was free to serve one coding to every client -- the exact bug this header exists to prevent, reintroduced by configuration. I argued earlier that this was simply the precedence "response_headers" has over every generated header, as with an overridden ETag. That was the wrong read of the precedent: #338 did not ignore the ETag interaction, it handled it, by skipping preconditions when the validator is overridden. The consistent treatment here is to handle it too. Mark the response when it was chosen by negotiation, and re-assert the header after "response_headers" has run. The merge is idempotent -- it leaves "*" alone, leaves a list already naming the header alone, and re-adds the field when it was removed -- so re-running it is enough. An operator writing "Vary: Origin" is almost always adding CORS and does not know Unit generates the field at all; taking that as an instruction to drop the coding from the cache key serves nobody. Measured, gzip configured for text/css: (no response_headers) -> Accept-Encoding Vary: Origin -> Origin, Accept-Encoding Vary: null -> Accept-Encoding Vary: * -> * Vary: Accept-Encoding -> Accept-Encoding Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- src/nxt_http.h | 9 +++++++++ src/nxt_http_compression.c | 8 +++++--- src/nxt_http_request.c | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/nxt_http.h b/src/nxt_http.h index 82ddd1498..fddc3f0c0 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -118,6 +118,14 @@ typedef struct { nxt_http_field_t *content_length; nxt_off_t content_length_n; const nxt_str_t *mime_type; + + /* + * The response was chosen by negotiation on Accept-Encoding, so its Vary + * must name that header. Kept on the response because "response_headers" + * is applied after the header is generated and may replace or remove it; + * nxt_http_request_header_send() re-asserts it once that has run. + */ + uint8_t vary_accept_encoding; /* 1 bit */ } nxt_http_response_t; @@ -492,6 +500,7 @@ nxt_int_t nxt_http_rewrite_init(nxt_router_conf_t *rtcf, nxt_int_t nxt_http_rewrite(nxt_task_t *task, nxt_http_request_t *r); nxt_bool_t nxt_http_set_headers_override_validators(nxt_http_request_t *r); +nxt_int_t nxt_http_comp_merge_vary(nxt_http_request_t *r); nxt_int_t nxt_http_set_headers_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf); nxt_int_t nxt_http_set_headers(nxt_http_request_t *r); diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index cab09e406..970979f10 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -623,8 +623,8 @@ nxt_http_comp_is_resp_content_encoded(const nxt_http_request_t *r) * them. */ -static nxt_int_t -nxt_http_comp_set_vary(nxt_http_request_t *r) +nxt_int_t +nxt_http_comp_merge_vary(nxt_http_request_t *r) { u_char *p, *end, *tok; nxt_int_t len; @@ -856,7 +856,9 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) * negotiation on Accept-Encoding, whichever coding is chosen below. */ - if (nxt_slow_path(nxt_http_comp_set_vary(r) != NXT_OK)) { + r->resp.vary_accept_encoding = 1; + + if (nxt_slow_path(nxt_http_comp_merge_vary(r) != NXT_OK)) { return NXT_ERROR; } diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index d20af82ef..9155642a1 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -696,6 +696,24 @@ nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } + /* + * "response_headers" has just run, and it replaces or removes a field it + * names outright. A response chosen by negotiation still varies on + * Accept-Encoding whatever an operator wrote there -- almost always they + * are adding Origin for CORS, unaware Unit generates the field at all -- + * so dropping it here would hand a shared cache the licence to serve one + * coding to every client. Re-assert it. The merge is idempotent: it + * leaves "*" alone, leaves a list that already names the header alone, + * and re-adds the field if it was removed. + */ + + if (r->resp.vary_accept_encoding) { + ret = nxt_http_comp_merge_vary(r); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + } + /* * RFC 9112 Sect. 6.3: a 1xx, 204 or 304 response, and any response to a * HEAD request, never has a message body, no matter what the response From 0a4d9a23e7521e9e00e94499e0a8aa7a178e9d96 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 13 Sep 2026 22:58:43 +0200 Subject: [PATCH 171/391] port: re-arm a port without allocating A write that cannot be sent now and cannot be held for later leaves the port's write event disarmed, and only port->engine may arm it again. A caller on another engine had to ask through nxt_port_post(), which allocates the work item it posts -- and that allocation fails under exactly the memory pressure that sent the write down this path, since the write got here because nxt_port_msg_insert_tail() could not allocate either. The failure was silent in the way that matters: nxt_port_error_handler() drains what the port holds, but nothing sets write_ready back and no event is armed, so every later message on that port was queued and never sent until the port closed. With "limits": {"timeout"} unset, requests routed there wait rather than fail. So the port carries the item. ->rearm_work is posted instead of an allocated one, and ->rearm_pending, taken from 0 to 1 by the poster and cleared by the handler, keeps it off the engine's queue twice: the queue links tail->next without checking, so one item posted twice becomes its own successor and nxt_locked_work_queue_add() spins. ->release_work could not be reused and its guarantee does not carry over. It is posted only by the thread that CASes the last reference away, so at most one is ever in flight by construction; nothing like that holds for a re-arm, which any thread can want at any time. What does carry over is the reference: the item takes one, and the handler drops it on port->engine, so the port cannot be released while the item is queued. The handler also checks pair[1]. A port can be closed while the item is in flight, and nxt_port_write_close() leaves socket.fd at the number it had, so arming would hand that number -- possibly somebody else's by then -- to this port's handlers. Every close runs on port->engine, so the read is stable. This was already reachable through the old nxt_port_post(nxt_port_fd_enable_write) path. A lost queue marker rides along, because it needs the same thing. When a payload reaches the shared ring and the READ_QUEUE marker cannot be written, the ring has taken the buffer, so the write must still answer NXT_OK -- but nothing tells the peer to look, and nxt_port_queue_send() raises "notify" only on the 0-to-1 transition, so no later enqueue announces it either. After EAGAIN the peer recovers by itself: that errno means the socket holds data it has not read, and nxt_port_queue_read_handler() drains the ring before the socket. After ENOBUFS it promises nothing. ->announce records that a marker is owed -- a fact about the port, not work to run, so a flag and not a second item. nxt_port_announce() sends the header from the stack with one sendmsg() and no allocation, and clears the flag only once it has gone. A marker that lands makes the peer drain the whole ring, so anything enqueued meanwhile is covered. The attempt is made from the re-arm handler rather than left to the write event, because after ENOBUFS the socket never became unwritable and there may be nothing for the write path to do; the handler is already running on the right engine, so it costs one sendmsg() to try. Three legs in src/test/nxt_port_fail_test.c, each red without its guard: the posted item is the port's own and a second re-arm does not queue it again ("a second re-arm posted the same item again (signals 2 use_count 3)"); a port closed while the item is in flight is not armed ("the re-arm enabled the write event of a closed port"); and the owed marker reaches the peer as a READ_QUEUE header once the socket drains ("the queue marker was still owed after the port became writable"). An owed marker is retried from every later pass through this function, not only from the one that finds the write event disarmed: the first retry runs in the re-arm handler, which arms the event, so nothing would ask again and a second ENOBUFS would strand the marker. nxt_port_fd_block_write() still goes through an allocating post whose return nobody reads. It is the same failure with the opposite effect -- the event stays armed on a full socket rather than disarmed on an empty one -- and it predates this change, so it is left to its own fix. ->rearm_pending counts posts, not re-arms. A caller already on port->engine does the work directly and touches neither the flag nor the reference count: decrementing a flag no CAS had incremented wrapped it -- nxt_atomic_t is unsigned -- and every later cross-engine post was then refused by the CAS, which would have stalled the port without needing an allocation failure at all. The inline leg asserts the flag is untouched after a same-engine re-arm, and the cross-engine leg asserts one in flight survives a same-engine re-arm beside it. Fixes: #350 Co-Authored-By: Claude Opus 5 --- CHANGES | 7 + docs/changes.xml | 11 ++ src/nxt_port.h | 26 ++++ src/nxt_port_socket.c | 206 ++++++++++++++++++++----- src/test/nxt_port_fail_test.c | 274 ++++++++++++++++++++++++++++++++++ 5 files changed, 483 insertions(+), 41 deletions(-) diff --git a/CHANGES b/CHANGES index 47fa8ee6d..9fc8bd502 100644 --- a/CHANGES +++ b/CHANGES @@ -116,6 +116,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 prototype that was not coming, and the requests waiting on it were never failed. + *) Bugfix: a port could stop sending after it ran out of memory. When a + write had to be retried later, the port asked its own thread to watch + the socket again, and that request needed memory of its own. If it + failed, nothing watched the socket and every later message waited in + the queue until the port closed. The port now carries the request it + needs, so it cannot fail for lack of memory. + *) Bugfix: the port layer reported a message as sent when it had been neither sent nor queued. This happened when a socket answered EAGAIN and no memory was left to hold the message for a later diff --git a/docs/changes.xml b/docs/changes.xml index abfda99c4..147cf07a0 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -131,6 +131,17 @@ requests waiting on it were never failed. + + +a port could stop sending after it ran out of memory. When a write had to be +retried later, the port asked its own thread to watch the socket again, and +that request needed memory of its own. If it failed, nothing watched the +socket and every later message waited in the queue until the port closed. The +port now carries the request it needs, so it cannot fail for lack of +memory. + + + the port layer reported a message as sent when it had been neither sent nor diff --git a/src/nxt_port.h b/src/nxt_port.h index 13ee4dec3..b2db9bb39 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -323,6 +323,31 @@ struct nxt_port_s { */ nxt_work_t release_work; + /* + * The write event has to be re-armed on port->engine, and a caller on + * another engine cannot do it directly. nxt_port_post() would allocate + * the item it posts, which fails exactly when the re-arm is needed most: + * the write that could not be held for a later attempt ran out of memory + * too. So the item lives here. + * + * ->release_work gets its uniqueness from the reference count -- only one + * thread can hand over the last reference -- and nothing like that holds + * for a re-arm, which any thread can want at any time. ->rearm_pending + * is that guarantee instead: the poster takes it from 0 to 1 and the + * handler clears it, so the item is on the engine's queue at most once. + * Re-arming twice would cost nothing, but linking the same item twice + * makes it its own successor. + * + * ->announce says the shared queue holds items whose READ_QUEUE marker + * was never sent. It is a fact about the port rather than work to run, + * so it is a flag and not a second item: any thread may set it, only + * port->engine clears it, and it is cleared only once a marker has gone + * out. + */ + nxt_work_t rearm_work; + nxt_atomic_t rearm_pending; + nxt_atomic_t announce; + nxt_buf_t *free_bufs; nxt_socket_t pair[2]; @@ -483,6 +508,7 @@ nxt_int_t nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, void nxt_port_change_log_file(nxt_task_t *task, nxt_runtime_t *rt, nxt_uint_t slot, nxt_fd_t fd); void nxt_port_remove_notify_others(nxt_task_t *task, nxt_process_t *process); +void nxt_port_rearm(nxt_task_t *task, nxt_port_t *port); void *nxt_port_queue_mmap(nxt_task_t *task, nxt_fd_t fd, size_t size); void nxt_port_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 9cdc13c06..bb67dc19d 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -37,6 +37,10 @@ static void nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg); static nxt_buf_t *nxt_port_buf_alloc(nxt_port_t *port); static void nxt_port_buf_free(nxt_port_t *port, nxt_buf_t *b); +static void nxt_port_announce(nxt_task_t *task, nxt_port_t *port); +static void nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port); +static void nxt_port_rearm_work_handler(nxt_task_t *task, void *obj, + void *data); static void nxt_port_error_handler(nxt_task_t *task, void *obj, void *data); #if (NXT_TESTS) @@ -378,16 +382,16 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, * * ENOBUFS promises nothing: nxt_socketpair_send() maps it to * NXT_AGAIN as well (src/nxt_socketpair.c), and it means the system - * ran out of buffers, not that the peer owes this port a read. The - * item then waits until something else makes that peer read. Keeping - * the notification instead needs one the port has already allocated, - * the way ->release_work is held for the release path; the alert - * marks the case until then. + * ran out of buffers, not that the peer owes this port a read. + * + * So record that a marker is owed and re-arm. nxt_port_announce() + * sends it from the stack with no allocation, either here on this + * port's own engine or from the item nxt_port_rearm() posts. */ - nxt_alert(task, "port{%d,%d} %d: the queued message could not be " - "announced; the peer takes it only when it next reads this " - "port", (int) port->pid, (int) port->id, port->socket.fd); + nxt_atomic_fetch_add(&port->announce, 1); + + nxt_port_rearm(task, port); res = NXT_OK; } @@ -510,10 +514,153 @@ nxt_port_fd_block_write(nxt_task_t *task, nxt_port_t *port, void *data) } +/* + * Tell the peer the shared queue has something in it, when the marker that + * should have said so could not be written. + * + * The marker is a bare header with no buffer and no descriptor, so it goes out + * from the stack with one sendmsg() and allocates nothing -- which is the + * point, because this runs after an allocation failure. The flag is cleared + * only once the marker has left: a peer that reads one drains the whole ring, + * so anything enqueued in between is covered by the same marker. + */ + static void -nxt_port_fd_enable_write(nxt_task_t *task, nxt_port_t *port, void *data) +nxt_port_announce(nxt_task_t *task, nxt_port_t *port) { + ssize_t n; + nxt_fd_t fd[2]; + struct iovec iov; + nxt_port_msg_t msg; + + /* + * pair[1], not socket.fd: nxt_port_write_close() clears the first and + * leaves the second at the number it had, so socket.fd tells you nothing + * about whether this port may still be written to. The caller checks the + * same thing before it arms the event; this repeats it because the test + * is cheap and the consequence of getting it wrong is writing to somebody + * else's descriptor. + */ + + if (port->announce == 0 || port->pair[1] == -1) { + return; + } + + /* nxt_socketpair_send() reads both, whether or not it sends them. */ + + fd[0] = -1; + fd[1] = -1; + + nxt_memzero(&msg, sizeof(nxt_port_msg_t)); + + msg.type = _NXT_PORT_MSG_READ_QUEUE; + msg.pid = nxt_pid; + msg.last = 1; + + iov.iov_base = &msg; + iov.iov_len = sizeof(nxt_port_msg_t); + + n = nxt_socketpair_send(&port->socket, fd, &iov, 1); + + if (n == (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_atomic_fetch_add(&port->announce, -1); + + nxt_debug(task, "port{%d,%d} %d: queue announced", (int) port->pid, + (int) port->id, port->socket.fd); + } +} + + +/* + * The work itself, on port->engine and touching neither the flag nor the + * reference count: both belong to whoever posted, and a caller already on + * this engine posts nothing. + * + * The descriptor is checked because a port can be closed while an item is in + * flight: nxt_port_write_close() sets pair[1] to -1 but leaves socket.fd at + * the number it had, so enabling would register that number -- possibly + * somebody else's by then -- against this port's handlers. Every close runs + * on port->engine, which is this thread, so the read is stable. + */ + +static void +nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port) +{ + if (port->pair[1] == -1) { + return; + } + nxt_fd_event_enable_write(task->thread->engine, &port->socket); + + nxt_port_announce(task, port); +} + + +/* + * Clear ->rearm_pending before the work, not after: a poster that arrives + * while this runs then posts again rather than dropping the re-arm, and + * enabling the same write event twice costs nothing. The reverse order can + * lose one, because a poster that sees the flag set cannot know whether the + * enable it skipped has happened yet. + * + * Only this handler clears it, and only because the post that queued this + * item set it. The flag counts posts, not re-arms. + */ + +static void +nxt_port_rearm_work_handler(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_t *port; + + port = obj; + + nxt_atomic_fetch_add(&port->rearm_pending, -1); + + nxt_port_rearm_now(task, port); + + /* + * The reference the post took. It runs on port->engine, so this drop + * takes the on-engine branch of nxt_port_use() and releases the port only + * if nobody else holds it -- the same discipline nxt_port_post() uses. + */ + + nxt_port_use(task, port, -1); +} + + +/* + * Re-arm the write event, and announce the queue if a marker is owed. + * + * On port->engine both are immediate. From another engine they have to be + * posted, and nxt_port_post() would allocate the item -- which fails exactly + * when this is needed, since the write that brought us here ran out of memory + * too. So the port carries its own item, and ->rearm_pending keeps it off the + * engine's queue twice: linking one item as its own successor would spin + * nxt_locked_work_queue_add() forever. + */ + +void +nxt_port_rearm(nxt_task_t *task, nxt_port_t *port) +{ + if (task->thread->engine == port->engine) { + nxt_port_rearm_now(task, port); + + return; + } + + if (!nxt_atomic_cmp_set(&port->rearm_pending, 0, 1)) { + return; + } + + nxt_atomic_fetch_add(&port->use_count, 1); + + port->rearm_work.handler = nxt_port_rearm_work_handler; + port->rearm_work.task = &port->engine->task; + port->rearm_work.obj = port; + port->rearm_work.data = NULL; + port->rearm_work.next = NULL; + + nxt_event_engine_post(port->engine, &port->rearm_work); } @@ -779,39 +926,16 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data) nxt_port_post(task, port, nxt_port_fd_block_write, NULL); } - if (enable_write) { - /* - * Same engine, nxt_port_post() is a direct call and cannot fail. - * Cross-engine, it allocates the work item it posts, and that can - * fail under the same memory pressure that sent this write down the - * refusal path above. A re-arm that never happens leaves - * write_ready at 0 with no event to set it again, so anything on - * this port's queue -- inserted concurrently, or by the next caller - * -- would wait forever. - * - * Raise the error handler instead, as every other failure here - * does. It drains the queue and completes what it holds, so those - * callers get their buffers back. - * - * It does not repair the port. write_ready stays 0 with no write - * event armed, so every later message is queued and never sent until - * the port is closed. An embedded work item on the port, the way - * ->release_work already works, would let the re-arm run without an - * allocation and remove this branch. - */ - - if (nxt_slow_path(nxt_port_post(task, port, nxt_port_fd_enable_write, - NULL) != NXT_OK)) - { - nxt_alert(task, "port{%d,%d} %d: cannot re-arm the write event; " - "draining what it holds, and it sends nothing further", - (int) port->pid, (int) port->id, port->socket.fd); - - use_delta++; + /* + * An owed marker is a reason to come back even when the event needs no + * arming: the first retry runs from the re-arm handler, which enables the + * event, so nothing would set enable_write again and a second ENOBUFS + * would strand the marker. Every later pass through this function tries + * once more. + */ - nxt_work_queue_add(wq, nxt_port_error_handler, task, - &port->socket, &port->socket); - } + if (enable_write || port->announce != 0) { + nxt_port_rearm(task, port); } if (use_delta != 0) { diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index b5e893f6d..69e827d40 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -26,6 +26,7 @@ static nxt_int_t nxt_port_fail_test_cross_engine_release(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_cross_engine_acquire(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_cross_engine_race(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_cross_engine_batch(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_cross_engine_rearm(nxt_thread_t *thr); static void nxt_port_fail_test_racer(void *data); static nxt_int_t nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, nxt_work_t *item); @@ -91,6 +92,10 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_cross_engine_rearm(thr) != NXT_OK) { + return NXT_ERROR; + } + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "port failure test passed"); @@ -269,6 +274,7 @@ static nxt_int_t nxt_port_fail_test_inline_drop(nxt_thread_t *thr) { int sndbuf; + ssize_t n; nxt_mp_t *mp; nxt_fd_t fd, pair[2]; nxt_buf_t *buf; @@ -508,6 +514,66 @@ nxt_port_fail_test_inline_drop(nxt_thread_t *thr) goto done; } + /* + * The marker that should have announced it could not be written, so the + * port owes one. Nothing has been lost: once the socket takes writes + * again, the re-arm sends the marker from the stack -- no allocation, + * which is the point, since an allocation is what failed. + */ + + if (port->announce != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the lost queue marker was not " + "recorded (announce %d)", (int) port->announce); + goto done; + } + + while (recv(pair[0], block, sizeof(block), MSG_DONTWAIT) > 0) { + /* Make room, as a peer that reads its backlog would. */ + } + + nxt_port_rearm(task, port); + + if (port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the queue marker was still owed " + "after the port became writable"); + goto done; + } + + /* + * Every re-arm in this leg ran on the port's own engine, so none of them + * posted anything and none may have touched the count of posts in flight. + * The flag is unsigned: one stray decrement wraps it to its maximum and + * every later cross-engine post is refused by the CAS, silently. + */ + + if (port->rearm_pending != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: a same-engine re-arm changed the " + "posted-item flag (rearm_pending %A)", + port->rearm_pending); + goto done; + } + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != (ssize_t) sizeof(nxt_port_msg_t)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the announcement was %d bytes, " + "expected a %d byte header", (int) n, + (int) sizeof(nxt_port_msg_t)); + goto done; + } + + if (((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the announcement was of type %d, " + "expected READ_QUEUE", + (int) ((nxt_port_msg_t *) block)->type); + goto done; + } + ret = NXT_OK; done: @@ -1422,6 +1488,214 @@ nxt_port_fail_test_racer(void *data) * which is exactly what the draining engine does. */ +/* + * Re-arming a port that belongs to another engine. + * + * nxt_port_rearm() posts the port's own item rather than allocating one, so + * the re-arm survives the memory pressure that made the write fail in the + * first place. Three things have to hold: the item on the queue is the + * port's, a second attempt does not link it twice, and the reference the post + * takes is given back by the handler. + * + * The last phase covers a port closed while the item is in flight. The + * handler must not enable the event then: nxt_port_write_close() leaves + * socket.fd at the number it had, which by that point may belong to somebody + * else. + */ + +static nxt_int_t +nxt_port_fail_test_cross_engine_rearm(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_work_t *posted; + nxt_event_engine_t current, foreign; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + + nxt_memzero(¤t, sizeof(current)); + nxt_work_queue_cache_create(¤t.work_queue_cache, 1024); + current.fast_work_queue.cache = ¤t.work_queue_cache; + nxt_work_queue_name(¤t.fast_work_queue, "fast"); + + nxt_memzero(&foreign, sizeof(foreign)); + foreign.task.thread = thr; + foreign.task.log = thr->log; + foreign.event.signal = nxt_port_fail_test_engine_signal; + foreign.event.enable_write = nxt_port_fail_test_enable_write; + + thr->engine = ¤t; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + port->engine = &foreign; + + /* Any descriptor but -1: the stub engine never touches the number. */ + port->pair[1] = 0; + + nxt_port_fail_test_signals = 0; + nxt_port_fail_test_rearms = 0; + + posted = &port->rearm_work; + + nxt_port_rearm(task, port); + + if (nxt_slow_path(nxt_port_fail_test_rearms != 0)) { + nxt_log_alert(thr->log, "port fail test: the cross-engine re-arm ran " + "on the calling thread"); + goto done; + } + + if (nxt_slow_path(foreign.locked_work_queue.head != posted)) { + nxt_log_alert(thr->log, "port fail test: the posted item is not the " + "port's embedded re-arm work"); + goto done; + } + + if (nxt_slow_path(nxt_port_fail_test_signals != 1)) { + nxt_log_alert(thr->log, "port fail test: the cross-engine re-arm did " + "not signal the target engine (%ui)", + nxt_port_fail_test_signals); + goto done; + } + + if (nxt_slow_path(port->use_count != 2 || port->rearm_pending != 1)) { + nxt_log_alert(thr->log, "port fail test: the posted re-arm holds no " + "reference (use_count %A pending %A)", port->use_count, + port->rearm_pending); + goto done; + } + + /* A second attempt while the first is queued must be a no-op. */ + + nxt_port_rearm(task, port); + + if (nxt_slow_path(nxt_port_fail_test_signals != 1 + || port->use_count != 2)) + { + nxt_log_alert(thr->log, "port fail test: a second re-arm posted the " + "same item again (signals %ui use_count %A)", + nxt_port_fail_test_signals, port->use_count); + goto done; + } + + /* The handler runs on the port's engine, so this thread becomes it. */ + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_rearms != 1)) { + nxt_log_alert(thr->log, "port fail test: the posted item re-armed the " + "write event %ui times, expected once", + nxt_port_fail_test_rearms); + goto done; + } + + if (nxt_slow_path(port->use_count != 1 || port->rearm_pending != 0)) { + nxt_log_alert(thr->log, "port fail test: the re-arm handler did not " + "give its reference back (use_count %A pending %A)", + port->use_count, port->rearm_pending); + goto done; + } + + /* + * A same-engine re-arm while a cross-engine post is still queued: it does + * its own work and leaves the queued item exactly as it was. Clearing the + * flag here would let a second poster link the same item again -- and + * severing its ->next would drop whatever the locked queue held behind it. + */ + + thr->engine = ¤t; + + nxt_port_rearm(task, port); + + thr->engine = &foreign; + + nxt_port_rearm(task, port); + + if (nxt_slow_path(port->rearm_pending != 1 + || foreign.locked_work_queue.head != posted + || posted->next != NULL)) + { + nxt_log_alert(thr->log, "port fail test: a same-engine re-arm " + "disturbed a post already in flight (pending %A)", + port->rearm_pending); + goto done; + } + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(port->rearm_pending != 0 || port->use_count != 1)) { + nxt_log_alert(thr->log, "port fail test: the queued post did not " + "settle (pending %A use_count %A)", port->rearm_pending, + port->use_count); + goto done; + } + + nxt_port_fail_test_rearms = 1; + + /* Closed while the next one is in flight: no enable, reference returned. */ + + thr->engine = ¤t; + + nxt_port_rearm(task, port); + + port->pair[1] = -1; + + thr->engine = &foreign; + + nxt_locked_work_queue_move(thr, &foreign.locked_work_queue, + ¤t.fast_work_queue); + + nxt_port_fail_test_drain_wq(¤t.fast_work_queue); + + if (nxt_slow_path(nxt_port_fail_test_rearms != 1)) { + nxt_log_alert(thr->log, "port fail test: the re-arm enabled the write " + "event of a closed port"); + goto done; + } + + if (nxt_slow_path(port->use_count != 1 || port->rearm_pending != 0)) { + nxt_log_alert(thr->log, "port fail test: a re-arm that found the port " + "closed kept its reference (use_count %A pending %A)", + port->use_count, port->rearm_pending); + goto done; + } + + ret = NXT_OK; + +done: + + if (port != NULL) { + /* nxt_port_mp_cleanup() asserts on this in a debug build. */ + port->pair[1] = -1; + + thr->engine = &foreign; + nxt_port_use(task, port, -1); + } + + thr->engine = ¤t; + + nxt_work_queue_cache_destroy(¤t.work_queue_cache); + + return ret; +} static nxt_int_t nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, nxt_work_t *item) { From eb7a64ccf3216a914b728a747d8a53e7c24e1b6d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 00:44:01 +0200 Subject: [PATCH 172/391] port: name the change the test wrapper refers to The comment on nxt_port_test_run_read_msg_process() cites "issue freeunitorg/freeunit#..." -- a placeholder that was never filled in, so it points at nothing. It also calls it an issue; it was a pull request. Mine, from the change it describes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011BvcUzwdMNFyRDBE1Exvjf --- src/nxt_port_socket.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index bb67dc19d..5c9d09046 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -79,8 +79,8 @@ nxt_port_test_run_error_handler(nxt_task_t *task, nxt_port_t *port) * -- used to verify the tail close of whatever a dispatch leaves in * msg->fd[], and the close in the last-fragment merge that runs before * msg->fd[0] is overwritten by the assembled message's own descriptors - * (issue freeunitorg/freeunit#... "port: close the descriptors a handler - * does not take"). + * (freeunitorg/freeunit#342, "port: close the descriptors a handler does + * not take"). */ void nxt_port_test_run_read_msg_process(nxt_task_t *task, nxt_port_t *port, From 76cf48ebef7e4a17734c6b95c0e7bfd4348d2f33 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 01:35:18 +0200 Subject: [PATCH 173/391] src: remove the unused nxt_dyld wrapper nxt_dyld_load(), nxt_dyld_symbol() and nxt_dyld_unload() wrap dlopen()/dlsym()/dlclose() with logging, but no code calls them: the language module loader in nxt_application.c calls dlopen() and dlsym() directly and never used this wrapper. The file dates to the initial import and was never wired up in the reachable history (932 commits). Unlike the fiber and dead-file removals in #336/#337, this file was compiled into libnxt.a, so libnxt.a loses one object file (nxt_dyld.o). Verified via a before/after object-list diff, a tree-wide search for nxt_dyld* and nxt_dyld_t, and a full build plus ./build/tests run (exit 0, 50 tests passed). Co-Authored-By: Claude Sonnet 5 --- CHANGES | 6 ++++ auto/sources | 1 - docs/changes.xml | 10 ++++++ src/nxt_dyld.c | 86 ------------------------------------------------ src/nxt_dyld.h | 29 ---------------- src/nxt_main.h | 1 - 6 files changed, 16 insertions(+), 117 deletions(-) delete mode 100644 src/nxt_dyld.c delete mode 100644 src/nxt_dyld.h diff --git a/CHANGES b/CHANGES index 9fc8bd502..b9a9b4482 100644 --- a/CHANGES +++ b/CHANGES @@ -100,6 +100,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 setjmp() branch that tested it could not run. The thread, log and sprintf sites were already disabled. Nothing observable changes. + *) Change: the nxt_dyld wrapper around dlopen()/dlsym()/dlclose() has + been removed. It shipped in the initial import and was never called: + the language module loader in src/nxt_application.c calls dlopen() + and dlsym() itself and never used this wrapper. libnxt.a loses one + object file; there are no other callers. + *) Bugfix: do not exit when the capget() syscall is denied. A syscall filter that omits capget -- a systemd "SystemCallFilter=" allowlist, a hand-written seccomp profile -- made a non-root unitd fail to start diff --git a/auto/sources b/auto/sources index aebe79819..8a002374f 100644 --- a/auto/sources +++ b/auto/sources @@ -23,7 +23,6 @@ NXT_LIB_SRCS=" \ src/nxt_port_memory.c \ src/nxt_port_rpc.c \ src/nxt_port.c \ - src/nxt_dyld.c \ src/nxt_random.c \ src/nxt_queue.c \ src/nxt_rbtree.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 147cf07a0..c3a9cb761 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -108,6 +108,16 @@ observable changes. + + +the nxt_dyld wrapper around dlopen()/dlsym()/dlclose() has been removed. +It shipped in the initial import and was never called: the language +module loader in src/nxt_application.c calls dlopen() and dlsym() itself +and never used this wrapper. libnxt.a loses one object file; there are +no other callers. + + + do not exit when the capget() syscall is denied. A syscall filter that diff --git a/src/nxt_dyld.c b/src/nxt_dyld.c deleted file mode 100644 index 63e6be145..000000000 --- a/src/nxt_dyld.c +++ /dev/null @@ -1,86 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -nxt_int_t -nxt_dyld_load(nxt_dyld_t *dyld) -{ - const char *err; - - dyld->handle = dlopen(dyld->name, RTLD_NOW | RTLD_GLOBAL); - - if (dyld->handle != NULL) { - nxt_thread_log_debug("dlopen(\"%s\")", dyld->name); - return NXT_OK; - } - - err = dlerror(); - if (err == NULL) { - err = "(null)"; - } - - nxt_thread_log_alert("dlopen(\"%s\") failed: %s", dyld->name, err); - - return NXT_ERROR; -} - - -void * -nxt_dyld_symbol(nxt_dyld_t *dyld, const char *symbol) -{ - void *handle, *s; - const char *name; - const char *err; - - if (dyld == NXT_DYLD_ANY) { - handle = RTLD_DEFAULT; - name = "RTLD_DEFAULT"; - - } else { - handle = dyld->handle; - name = dyld->name; - } - - s = dlsym(handle, symbol); - - if (s != NULL) { - nxt_thread_log_debug("dlsym(\"%s\", \"%s\")", name, symbol); - return s; - } - - err = dlerror(); - if (err == NULL) { - err = "(null)"; - } - - nxt_thread_log_alert("dlsym(\"%s\", \"%s\") failed: %s", name, symbol, err); - - return s; -} - - -nxt_int_t -nxt_dyld_unload(nxt_dyld_t *dyld) -{ - const char *err; - - if (dlclose(dyld->handle) == 0) { - nxt_thread_log_debug("dlclose(\"%s\")", dyld->name); - return NXT_OK; - } - - err = dlerror(); - - if (err == NULL) { - err = "(null)"; - } - - nxt_thread_log_alert("dlclose(\"%s\") failed: %s", dyld->name, err); - - return NXT_ERROR; -} diff --git a/src/nxt_dyld.h b/src/nxt_dyld.h deleted file mode 100644 index 65ce1874a..000000000 --- a/src/nxt_dyld.h +++ /dev/null @@ -1,29 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_UNIX_DYLD_H_INCLUDED_ -#define _NXT_UNIX_DYLD_H_INCLUDED_ - - -typedef struct { - void *handle; - char *name; -} nxt_dyld_t; - - -#define NXT_DYLD_ANY RTLD_DEFAULT - - -#define nxt_dyld_is_valid(dyld) \ - ((dyld)->handle != NULL) - - -NXT_EXPORT nxt_int_t nxt_dyld_load(nxt_dyld_t *dyld); -NXT_EXPORT void *nxt_dyld_symbol(nxt_dyld_t *dyld, const char *symbol); -NXT_EXPORT nxt_int_t nxt_dyld_unload(nxt_dyld_t *dyld); - - -#endif /* _NXT_UNIX_DYLD_H_INCLUDED_ */ diff --git a/src/nxt_main.h b/src/nxt_main.h index c0731ea7a..f91a5fa99 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -82,7 +82,6 @@ typedef struct nxt_sockaddr_s nxt_sockaddr_t; #include #include #include -#include typedef void *(*nxt_mem_proto_alloc_t)(void *pool, size_t size); From cba15d60a32c9132d0a37e13e89f98de16a8cb1b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 01:38:06 +0200 Subject: [PATCH 174/391] build: remove unused src/nxt_queue.c nxt_queue.c defines only nxt_queue_sort() and nxt_queue_middle(), both NXT_EXPORTed. Neither has a caller anywhere in the tree (src, auto, fuzzing, test, tools, go, and the language runtimes), and a full-history grep back to the initial commit finds no call sites, only the definition and the header declarations -- the unused half of a queue utility library carried over from nginx, not scaffolding for a feature in progress. src/nxt_queue.h keeps every nxt_queue_* macro, used throughout the tree and unaffected by this change. The two now-dead NXT_EXPORT prototypes for nxt_queue_middle() and nxt_queue_sort() are removed with the functions: a declaration with no definition in a public header is a link-error trap for the next caller, not a harmless leftover. Verified via ./configure --tests && make && make tests: build and test binaries link clean with libnxt.a one translation unit smaller (build/src/nxt_queue.o gone, nothing else missing), and ./build/tests exits 0 with no FAILED lines. Co-Authored-By: Claude Sonnet 5 --- CHANGES | 6 ++++ auto/sources | 1 - docs/changes.xml | 9 +++++ src/nxt_queue.c | 85 ------------------------------------------------ src/nxt_queue.h | 6 ---- 5 files changed, 15 insertions(+), 92 deletions(-) delete mode 100644 src/nxt_queue.c diff --git a/CHANGES b/CHANGES index b9a9b4482..cd95bf479 100644 --- a/CHANGES +++ b/CHANGES @@ -215,6 +215,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 tools/unitc is unaffected: it still converts configuration to and from YAML, using yq. + *) Change: removed src/nxt_queue.c, the unused half of a queue utility + library carried over from nginx. It defined nxt_queue_sort() and + nxt_queue_middle(), both exported but never called anywhere in the + tree; src/nxt_queue.h and the nxt_queue_* macros it provides are + unaffected and untouched. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/auto/sources b/auto/sources index 8a002374f..5f9776d0b 100644 --- a/auto/sources +++ b/auto/sources @@ -24,7 +24,6 @@ NXT_LIB_SRCS=" \ src/nxt_port_rpc.c \ src/nxt_port.c \ src/nxt_random.c \ - src/nxt_queue.c \ src/nxt_rbtree.c \ src/nxt_mp.c \ src/nxt_mem_zone.c \ diff --git a/docs/changes.xml b/docs/changes.xml index c3a9cb761..87cf7e264 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -285,6 +285,15 @@ unaffected, and a build with default features behaves as before. + + +removed src/nxt_queue.c, the unused half of a queue utility library carried +over from nginx. It defined nxt_queue_sort() and nxt_queue_middle(), both +exported but never called anywhere in the tree; src/nxt_queue.h and the +nxt_queue_* macros it provides are unaffected and untouched. + + + diff --git a/src/nxt_queue.c b/src/nxt_queue.c deleted file mode 100644 index c81356dbd..000000000 --- a/src/nxt_queue.c +++ /dev/null @@ -1,85 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - - -#include - - -/* - * Find the middle queue element if the queue has odd number of elements, - * or the first element of the queue's second part otherwise. - */ - -nxt_queue_link_t * -nxt_queue_middle(nxt_queue_t *queue) -{ - nxt_queue_link_t *middle, *next; - - middle = nxt_queue_first(queue); - - if (middle == nxt_queue_last(queue)) { - return middle; - } - - next = middle; - - for ( ;; ) { - middle = nxt_queue_next(middle); - - next = nxt_queue_next(next); - - if (next == nxt_queue_last(queue)) { - return middle; - } - - next = nxt_queue_next(next); - - if (next == nxt_queue_last(queue)) { - return middle; - } - } -} - - -/* - * nxt_queue_sort() provides a stable sort because it uses the insertion - * sort algorithm. Its worst and average computational complexity is O^2. - */ - -void -nxt_queue_sort(nxt_queue_t *queue, - nxt_int_t (*cmp)(const void *data, const nxt_queue_link_t *, - const nxt_queue_link_t *), const void *data) -{ - nxt_queue_link_t *link, *prev, *next; - - link = nxt_queue_first(queue); - - if (link == nxt_queue_last(queue)) { - return; - } - - for (link = nxt_queue_next(link); - link != nxt_queue_tail(queue); - link = next) - { - prev = nxt_queue_prev(link); - next = nxt_queue_next(link); - - nxt_queue_remove(link); - - do { - if (cmp(data, prev, link) <= 0) { - break; - } - - prev = nxt_queue_prev(prev); - - } while (prev != nxt_queue_head(queue)); - - nxt_queue_insert_after(prev, link); - } -} diff --git a/src/nxt_queue.h b/src/nxt_queue.h index 6b7f5d577..4fa8eaac5 100644 --- a/src/nxt_queue.h +++ b/src/nxt_queue.h @@ -194,12 +194,6 @@ typedef struct { nxt_container_of(lnk, type, link) -NXT_EXPORT nxt_queue_link_t *nxt_queue_middle(nxt_queue_t *queue); -NXT_EXPORT void nxt_queue_sort(nxt_queue_t *queue, - nxt_int_t (*cmp)(const void *, const nxt_queue_link_t *, - const nxt_queue_link_t *), const void *data); - - #define nxt_queue_each(elt, queue, type, link) \ do { \ nxt_queue_link_t *_lnk, *_nxt; \ From 7d67c984de318cdec17c9cb8ec95821c5e4add4f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 01:38:44 +0200 Subject: [PATCH 175/391] Remove the unused job cluster nxt_job.c/.h wrap nxt_thread_pool_t as a generic deferred-work abstraction: nxt_job_start() posts to a job's thread_pool when one is set, otherwise it just runs the handler inline. Its only consumer, nxt_event_conn_job_sendfile.c, would offload a blocking sendfile() to that thread pool by setting jbs->job.thread_pool = c->u.thread_pool. Neither half was ever wired up. nxt_event_conn_job_sendfile() itself has zero callers anywhere in the tree (checked auto/, fuzzing/, test/, src/test/, every language module dir, and the sendfile platform conditionals in auto/sendfile and auto/os/* -- none reference it, and it is not behind any NXT_HAVE_*_SENDFILE branch). c->u.thread_pool is never assigned anywhere, so even a caller would always hand nxt_job a NULL thread pool and take the inline path. The file dates to the initial NGINX Unit import and was never touched again; nginx/unit's own PR/issue archive has no discussion of nxt_job or job-based sendfile, only of the unrelated per-platform sendfile() syscall wrappers (nxt_linux_sendfile.c and friends, which are unaffected and stay). This is the same design space as the thread-pool-offload idea behind the fiber subsystem removed in #336, but it is not a working alternative to it: the thread pool infrastructure it depends on (nxt_thread_pool_create/_post) is still created and used elsewhere at runtime, this file just never plugged sendfile into it. Removing it does not discard a live "blocking work" story -- there wasn't one here to begin with. Also drops the now-unused nxt_thread_pool_t *thread_pool member from nxt_conn_t's "u" union: nothing ever assigned it, and after removing its only reader (the sendfile job code) nothing reads it either. Every remaining reference to that union (nxt_conn_write.c, nxt_h1proto.c, nxt_openssl.c -- 13 references total) touches only u.tls, and all of them are already guarded by #if (NXT_TLS) or, for nxt_openssl.c, compiled only in a TLS build. So the union itself is now guarded by member at all, which matches what the tree actually uses. (Unrelated, pre-existing: nxt_gnutls.c already references c->u.ssltls, a field that was never a member of this union even before this change -- that mismatch predates this commit and is left untouched.) In a TLS build the union's only member is unchanged (void *tls), so nxt_conn_t's layout there is unchanged too. Verified with two configurations: - default (non-TLS): ./configure --tests && make && make tests all succeed; ./build/tests exits 0 with 49/49 "test passed" and 0 failures (redirected to a file, not piped). - ./configure --tests --openssl (TLS): same, all exit 0, 49/49 passed, 0 failures. The object list loses exactly nxt_job.o and nxt_event_conn_job_sendfile.o and nothing else; git grep for nxt_job, nxt_event_conn_job_sendfile, and u.thread_pool is empty afterwards. Co-Authored-By: Claude Sonnet 5 --- CHANGES | 6 + auto/sources | 2 - docs/changes.xml | 9 ++ src/nxt_conn.h | 8 +- src/nxt_event_conn_job_sendfile.c | 259 ------------------------------ src/nxt_job.c | 199 ----------------------- src/nxt_job.h | 86 ---------- src/nxt_main.h | 1 - 8 files changed, 17 insertions(+), 553 deletions(-) delete mode 100644 src/nxt_event_conn_job_sendfile.c delete mode 100644 src/nxt_job.c delete mode 100644 src/nxt_job.h diff --git a/CHANGES b/CHANGES index cd95bf479..4f443a00e 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 response is the one a cache must not reuse for a client that accepts gzip. + *) Change: removed the unused job cluster (nxt_job.c, nxt_job.h, and + nxt_event_conn_job_sendfile.c). The offload path they implemented + had no caller anywhere in the tree and its thread pool handoff was + structurally unreachable, since the connection field it read was + never assigned. + *) Feature: byte-range requests for static files. A "share" action now answers a satisfiable single-range "Range: bytes=..." request with "206 Partial Content" and a matching "Content-Range", and an diff --git a/auto/sources b/auto/sources index 5f9776d0b..bab4829ae 100644 --- a/auto/sources +++ b/auto/sources @@ -64,9 +64,7 @@ NXT_LIB_SRCS=" \ src/nxt_conn_read.c \ src/nxt_conn_write.c \ src/nxt_conn_close.c \ - src/nxt_event_conn_job_sendfile.c \ src/nxt_conn_proxy.c \ - src/nxt_job.c \ src/nxt_sockaddr.c \ src/nxt_listen_socket.c \ src/nxt_upstream.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 87cf7e264..11ed1c45b 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,15 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +removed the unused job cluster (nxt_job.c, nxt_job.h, and +nxt_event_conn_job_sendfile.c). The offload path they implemented had no +caller anywhere in the tree and its thread pool handoff was structurally +unreachable, since the connection field it read was never assigned. + + + byte-range requests for static files. A "share" action answers a "Range" diff --git a/src/nxt_conn.h b/src/nxt_conn.h index f2ef4932f..e146a05b2 100644 --- a/src/nxt_conn.h +++ b/src/nxt_conn.h @@ -146,12 +146,11 @@ struct nxt_conn_s { nxt_conn_io_t *io; - union { #if (NXT_TLS) + union { void *tls; -#endif - nxt_thread_pool_t *thread_pool; } u; +#endif nxt_mp_t *mem_pool; @@ -289,9 +288,6 @@ ssize_t nxt_event_conn_io_writev(nxt_conn_t *c, nxt_iobuf_t *iob, nxt_uint_t niob); ssize_t nxt_event_conn_io_send(nxt_conn_t *c, void *buf, size_t size); -NXT_EXPORT void nxt_event_conn_job_sendfile(nxt_task_t *task, - nxt_conn_t *c); - #define nxt_conn_connect(engine, c) \ nxt_work_queue_add(&engine->socket_work_queue, nxt_conn_sys_socket, \ diff --git a/src/nxt_event_conn_job_sendfile.c b/src/nxt_event_conn_job_sendfile.c deleted file mode 100644 index 0f6f9353a..000000000 --- a/src/nxt_event_conn_job_sendfile.c +++ /dev/null @@ -1,259 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -typedef struct { - nxt_job_t job; - nxt_buf_t *out; - size_t sent; - size_t limit; - nxt_work_handler_t ready_handler; -} nxt_job_sendfile_t; - - -static void nxt_event_conn_job_sendfile_start(nxt_task_t *task, void *obj, - void *data); -static void nxt_event_conn_job_sendfile_handler(nxt_task_t *task, void *obj, - void *data); -static void nxt_event_conn_job_sendfile_return(nxt_task_t *task, void *obj, - void *data); -static nxt_buf_t *nxt_event_conn_job_sendfile_completion(nxt_task_t *task, - nxt_conn_t *c, nxt_buf_t *b); - - -void -nxt_event_conn_job_sendfile(nxt_task_t *task, nxt_conn_t *c) -{ - nxt_fd_event_disable(task->thread->engine, &c->socket); - - /* A work item data is not used in nxt_event_conn_job_sendfile_start(). */ - nxt_event_conn_job_sendfile_start(task, c, NULL); -} - - -static void -nxt_event_conn_job_sendfile_start(nxt_task_t *task, void *obj, void *data) -{ - nxt_conn_t *c; - nxt_iobuf_t b; - nxt_job_sendfile_t *jbs; - nxt_sendbuf_coalesce_t sb; - - c = obj; - - nxt_debug(task, "event conn sendfile fd:%d", c->socket.fd); - - jbs = nxt_job_create(c->mem_pool, sizeof(nxt_job_sendfile_t)); - - if (nxt_slow_path(jbs == NULL)) { - c->write_state->error_handler(task, c, NULL); - return; - } - - c->socket.write_handler = nxt_event_conn_job_sendfile_start; - c->socket.error_handler = c->write_state->error_handler; - - jbs->job.data = c; - nxt_job_set_name(&jbs->job, "job sendfile"); - - jbs->limit = nxt_event_conn_write_limit(c); - - if (jbs->limit != 0) { - - sb.buf = c->write; - sb.iobuf = &b; - sb.nmax = 1; - sb.sync = 0; - sb.size = 0; - sb.limit = jbs->limit; - - if (nxt_sendbuf_mem_coalesce(c->socket.task, &sb) != 0 || !sb.sync) { - - jbs->job.thread_pool = c->u.thread_pool; - jbs->job.log = c->socket.log; - jbs->out = c->write; - c->write = NULL; - jbs->ready_handler = nxt_event_conn_job_sendfile_return; - - c->block_read = 1; - c->block_write = 1; - - nxt_job_start(task, &jbs->job, nxt_event_conn_job_sendfile_handler); - return; - } - } - - nxt_event_conn_job_sendfile_return(task, jbs, c); -} - - -static void -nxt_event_conn_job_sendfile_handler(nxt_task_t *task, void *obj, void *data) -{ - ssize_t ret; - nxt_buf_t *b; - nxt_bool_t first; - nxt_conn_t *c; - nxt_job_sendfile_t *jbs; - - jbs = obj; - c = data; - - nxt_debug(task, "event conn job sendfile fd:%d", c->socket.fd); - - first = c->socket.write_ready; - b = jbs->out; - - do { - ret = c->io->old_sendbuf(c, b, jbs->limit); - - if (ret == NXT_AGAIN) { - break; - } - - if (nxt_slow_path(ret == NXT_ERROR)) { - goto done; - } - - jbs->sent += ret; - jbs->limit -= ret; - - b = nxt_sendbuf_update(b, ret); - - if (b == NULL) { - goto done; - } - - if (jbs->limit == 0) { - - if (c->rate == NULL) { - jbs->limit = c->max_chunk; - goto fast; - } - - goto done; - } - - } while (c->socket.write_ready); - - if (first && task->thread->thread_pool->work_queue.head != NULL) { - goto fast; - } - -done: - - nxt_job_return(task, &jbs->job, jbs->ready_handler); - return; - -fast: - - nxt_work_set(&jbs->job.work, nxt_event_conn_job_sendfile_handler, - jbs->job.task, jbs, c); - - nxt_thread_pool_post(task->thread->thread_pool, &jbs->job.work); -} - - -static void -nxt_event_conn_job_sendfile_return(nxt_task_t *task, void *obj, void *data) -{ - size_t sent; - nxt_buf_t *b; - nxt_bool_t done; - nxt_conn_t *c; - nxt_job_sendfile_t *jbs; - - jbs = obj; - c = data; - - c->block_read = 0; - c->block_write = 0; - - sent = jbs->sent; - c->sent += sent; - - nxt_debug(task, "event conn sendfile sent:%z", sent); - - b = jbs->out; - - /* The job must be destroyed before connection error handler. */ - nxt_job_destroy(task, jbs); - - if (0 /* STUB: c->write_state->process_buffers */) { - b = nxt_event_conn_job_sendfile_completion(task, c, b); - - done = (b == NULL); - - /* Add data which might be added after sendfile job has started. */ - nxt_buf_chain_add(&b, c->write); - c->write = b; - - if (done) { - /* All data has been sent. */ - - if (b != NULL) { - /* But new data has been added. */ - nxt_event_conn_job_sendfile_start(task, c, NULL); - } - - return; - } - } - - if (sent != 0 && c->write_state->timer_autoreset) { - nxt_timer_disable(task->thread->engine, &c->write_timer); - } - - if (c->socket.error == 0 - && !nxt_event_conn_write_delayed(task->thread->engine, c, sent)) - { - nxt_conn_timer(task->thread->engine, c, c->write_state, - &c->write_timer); - - nxt_fd_event_oneshot_write(task->thread->engine, &c->socket); - } - - if (sent != 0) { - nxt_work_queue_add(c->write_work_queue, c->write_state->ready_handler, - task, c, c->socket.data); - /* - * Fall through if first operations were - * successful but the last one failed. - */ - } - - if (nxt_slow_path(c->socket.error != 0)) { - nxt_work_queue_add(c->write_work_queue, c->write_state->error_handler, - task, c, c->socket.data); - } -} - - -static nxt_buf_t * -nxt_event_conn_job_sendfile_completion(nxt_task_t *task, nxt_conn_t *c, - nxt_buf_t *b) -{ - while (b != NULL) { - - nxt_prefetch(b->next); - - if (nxt_buf_is_mem(b) && b->mem.pos != b->mem.free) { - break; - - } else if (nxt_buf_is_file(b) && b->file_pos != b->file_end) { - break; - } - - nxt_work_queue_add(c->write_work_queue, - b->completion_handler, task, b, b->parent); - - b = b->next; - } - - return b; -} diff --git a/src/nxt_job.c b/src/nxt_job.c deleted file mode 100644 index 560739537..000000000 --- a/src/nxt_job.c +++ /dev/null @@ -1,199 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -static void nxt_job_thread_trampoline(nxt_task_t *task, void *obj, void *data); -static void nxt_job_thread_return_handler(nxt_task_t *task, void *obj, - void *data); - - -void * -nxt_job_create(nxt_mp_t *mp, size_t size) -{ - size_t cache_size; - nxt_job_t *job; - - if (mp == NULL) { - mp = nxt_mp_create(1024, 128, 256, 32); - if (nxt_slow_path(mp == NULL)) { - return NULL; - } - - job = nxt_mp_zget(mp, size); - cache_size = 0; - - } else { - job = nxt_mp_zalloc(mp, size); - cache_size = size; - } - - if (nxt_slow_path(job == NULL)) { - return NULL; - } - - job->cache_size = (uint16_t) cache_size; - job->mem_pool = mp; - nxt_job_set_name(job, "job"); - - /* Allow safe nxt_queue_remove() in nxt_job_destroy(). */ - nxt_queue_self(&job->link); - - return job; -} - - -void -nxt_job_init(nxt_job_t *job, size_t size) -{ - nxt_memzero(job, size); - - nxt_job_set_name(job, "job"); - - nxt_queue_self(&job->link); -} - - -void -nxt_job_destroy(nxt_task_t *task, void *data) -{ - nxt_job_t *job; - - job = data; - - nxt_queue_remove(&job->link); - - if (job->cache_size == 0) { - - if (job->mem_pool != NULL) { - nxt_mp_destroy(job->mem_pool); - } - - } else { - nxt_mp_free(job->mem_pool, job); - } -} - - -#if 0 - -nxt_int_t -nxt_job_cleanup_add(nxt_mp_t *mp, nxt_job_t *job) -{ - nxt_mem_pool_cleanup_t *mpcl; - - mpcl = nxt_mem_pool_cleanup(mp, 0); - - if (nxt_fast_path(mpcl != NULL)) { - mpcl->handler = nxt_job_destroy; - mpcl->data = job; - return NXT_OK; - } - - return NXT_ERROR; -} - -#endif - - -/* - * The (void *) casts in nxt_thread_pool_post() and nxt_event_engine_post() - * calls and to the "nxt_work_handler_t" are required by Sun C. - */ - -void -nxt_job_start(nxt_task_t *task, nxt_job_t *job, nxt_work_handler_t handler) -{ - nxt_debug(task, "%s start", job->name); - - if (job->thread_pool != NULL) { - nxt_int_t ret; - - job->engine = task->thread->engine; - - nxt_work_set(&job->work, nxt_job_thread_trampoline, - job->task, job, (void *) handler); - - ret = nxt_thread_pool_post(job->thread_pool, &job->work); - - if (ret == NXT_OK) { - return; - } - - handler = job->abort_handler; - } - - handler(job->task, job, job->data); -} - - -/* A trampoline function is called by a thread pool thread. */ - -static void -nxt_job_thread_trampoline(nxt_task_t *task, void *obj, void *data) -{ - nxt_job_t *job; - nxt_work_handler_t handler; - - job = obj; - handler = (nxt_work_handler_t) data; - - nxt_debug(task, "%s thread", job->name); - - if (nxt_slow_path(job->cancel)) { - nxt_job_return(task, job, job->abort_handler); - - } else { - handler(job->task, job, job->data); - } -} - - -void -nxt_job_return(nxt_task_t *task, nxt_job_t *job, nxt_work_handler_t handler) -{ - nxt_debug(task, "%s return", job->name); - - if (job->engine != NULL) { - /* A return function is called in thread pool thread context. */ - - nxt_work_set(&job->work, nxt_job_thread_return_handler, - job->task, job, (void *) handler); - - nxt_event_engine_post(job->engine, &job->work); - - return; - } - - if (nxt_slow_path(job->cancel)) { - nxt_debug(task, "%s cancellation", job->name); - handler = job->abort_handler; - } - - nxt_work_queue_add(&task->thread->engine->fast_work_queue, - handler, job->task, job, job->data); -} - - -static void -nxt_job_thread_return_handler(nxt_task_t *task, void *obj, void *data) -{ - nxt_job_t *job; - nxt_work_handler_t handler; - - job = obj; - handler = (nxt_work_handler_t) data; - - job->task->thread = task->thread; - - if (nxt_slow_path(job->cancel)) { - nxt_debug(task, "%s cancellation", job->name); - handler = job->abort_handler; - } - - handler(job->task, job, job->data); -} diff --git a/src/nxt_job.h b/src/nxt_job.h deleted file mode 100644 index 0495c4844..000000000 --- a/src/nxt_job.h +++ /dev/null @@ -1,86 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_JOB_H_INCLUDED_ -#define _NXT_JOB_H_INCLUDED_ - - -/* - * A job may run by separate thread, so each job should have its - * its own mem_pool. A job itself is allocated from this mem_pool. - * On job completion a job initiator can destroy the job at once - * with nxt_job_destroy() or can postpone the destruction with - * nxt_job_cleanup_add(), if the initiator uses data from the job's - * mem_pool. - * - * Several child jobs may run in context of another job in the same - * thread. In this case the child job may use a mem_pool of the - * parent job and the child job is allocated using the mem_pool's cache. - * nxt_job_destroy() just returns the job to the cache. All job - * allocations however still remain in the parent mem_pool. - * - * The first thread in job thread pool is created on demand. If this - * operation fails the job abort handler is called. It also is called - * if the job is canceled. To avoid race condition the abort handler - * always runs in context of a thread initiated the job. The abort - * handler may be as simple as nxt_job_destroy(). - */ - - -typedef struct { - void *data; - - nxt_task_t *task; - - nxt_work_handler_t abort_handler; - - uint16_t cache_size; - uint8_t cancel; /* 1 bit */ - - nxt_mp_t *mem_pool; - nxt_queue_link_t link; - - nxt_thread_pool_t *thread_pool; - nxt_event_engine_t *engine; - nxt_log_t *log; - - nxt_work_t work; - -#if (NXT_DEBUG) - const char *name; -#endif - -} nxt_job_t; - - -NXT_EXPORT void *nxt_job_create(nxt_mp_t *mp, size_t size); -NXT_EXPORT void nxt_job_init(nxt_job_t *job, size_t size); -NXT_EXPORT void nxt_job_destroy(nxt_task_t *task, void *data); -NXT_EXPORT nxt_int_t nxt_job_cleanup_add(nxt_mp_t *mp, nxt_job_t *job); - -NXT_EXPORT void nxt_job_start(nxt_task_t *task, nxt_job_t *job, - nxt_work_handler_t handler); -NXT_EXPORT void nxt_job_return(nxt_task_t *task, nxt_job_t *job, - nxt_work_handler_t handler); - - -#define nxt_job_cancel(job) \ - (job)->cancel = 1 - - -#if (NXT_DEBUG) - -#define nxt_job_set_name(job, text) \ - (job)->name = text - -#else - -#define nxt_job_set_name(job, text) - -#endif - - -#endif /* _NXT_JOB_H_INCLUDED_ */ diff --git a/src/nxt_main.h b/src/nxt_main.h index f91a5fa99..d3634f017 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -144,7 +144,6 @@ typedef void (*nxt_event_conn_handler_t)(nxt_thread_t *thr, nxt_conn_t *c); #include #include -#include #include From fc5f08e10118e51928a5e69dfc0a367c957f3abf Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 01:42:10 +0200 Subject: [PATCH 176/391] Remove nxt_mem_zone, an unused second allocator nxt_mem_zone is a zone allocator over a mapped region. Its only caller in the tree was its own unit test (src/test/nxt_mem_zone_test.c), driven by src/test/nxt_tests.c. No product code -- router, application process, or any language module -- calls it; nxt_mp is the pool allocator the product actually uses, and its own test (nxt_mp_test) runs immediately before the removed mem_zone tests in the same driver. Upstream nginx/unit reached the same conclusion independently and deleted this file (commit f662c394, "Remove unused subsystems") on a branch that predates this fork's current master. Removing the allocator means removing its test, which drops the suite from 49 to 46 "test passed" lines (three "mem zone test passed" entries). The remaining 46 entries pass. Object list shrinks by exactly nxt_mem_zone.o and its test's nxt_mem_zone_test.o. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0143ggPMsUKpRPWoQLTxBZUk --- CHANGES | 10 + auto/sources | 2 - docs/changes.xml | 12 + src/nxt_main.h | 1 - src/nxt_mem_zone.c | 948 ----------------------------------- src/nxt_mem_zone.h | 28 -- src/test/nxt_mem_zone_test.c | 74 --- src/test/nxt_tests.c | 12 - src/test/nxt_tests.h | 2 - 9 files changed, 22 insertions(+), 1067 deletions(-) delete mode 100644 src/nxt_mem_zone.c delete mode 100644 src/nxt_mem_zone.h delete mode 100644 src/test/nxt_mem_zone_test.c diff --git a/CHANGES b/CHANGES index 4f443a00e..afecf2f6f 100644 --- a/CHANGES +++ b/CHANGES @@ -227,6 +227,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 tree; src/nxt_queue.h and the nxt_queue_* macros it provides are unaffected and untouched. + *) Change: nxt_mem_zone, a second allocator, is removed. It was a + zone allocator over a mapped region, distinct from nxt_mp, the + pool allocator product code actually uses; nothing in the router, + applications, or any language module called it. Its only caller + was its own unit test, run by "./build/tests"; that test is + removed too, dropping the three "mem zone test passed" lines from + the suite's output. Upstream nginx/unit had already reached the + same conclusion and deleted the file as part of an unrelated + cleanup. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/auto/sources b/auto/sources index bab4829ae..ad9fbfce2 100644 --- a/auto/sources +++ b/auto/sources @@ -26,7 +26,6 @@ NXT_LIB_SRCS=" \ src/nxt_random.c \ src/nxt_rbtree.c \ src/nxt_mp.c \ - src/nxt_mem_zone.c \ src/nxt_string.c \ src/nxt_utf8.c \ src/nxt_parse.c \ @@ -158,7 +157,6 @@ NXT_TEST_SRCS=" \ src/test/nxt_term_parse_test.c \ src/test/nxt_msec_diff_test.c \ src/test/nxt_mp_test.c \ - src/test/nxt_mem_zone_test.c \ src/test/nxt_lvlhsh_test.c \ src/test/nxt_gmtime_test.c \ src/test/nxt_sprintf_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 11ed1c45b..0b381c4f8 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -303,6 +303,18 @@ nxt_queue_* macros it provides are unaffected and untouched. + + +nxt_mem_zone, a second allocator, is removed. It was a zone allocator over a +mapped region, distinct from nxt_mp, the pool allocator product code actually +uses; nothing in the router, applications, or any language module called it. +Its only caller was its own unit test, run by "./build/tests"; that test is +removed too, dropping the three "mem zone test passed" lines from the suite's +output. Upstream nginx/unit had already reached the same conclusion and +deleted the file as part of an unrelated cleanup. + + + diff --git a/src/nxt_main.h b/src/nxt_main.h index d3634f017..ab10e6848 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -93,7 +93,6 @@ typedef struct { } nxt_mem_proto_t; -#include #include #include diff --git a/src/nxt_mem_zone.c b/src/nxt_mem_zone.c deleted file mode 100644 index a3ba37007..000000000 --- a/src/nxt_mem_zone.c +++ /dev/null @@ -1,948 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include - - -#define NXT_MEM_ZONE_PAGE_FREE 0 -/* - * A page was never allocated before so it should be filled with - * junk on the first time allocation if memory debugging is enabled. - */ -#define NXT_MEM_ZONE_PAGE_FRESH 1 - -/* An entire page is currently used, no chunks inside the page. */ -#define NXT_MEM_ZONE_PAGE_USED 2 - - -typedef struct nxt_mem_zone_page_s nxt_mem_zone_page_t; - -struct nxt_mem_zone_page_s { - /* - * A size of page chunks if value is greater than or equal to 16. - * Otherwise it is used to mark page state: NXT_MEM_ZONE_PAGE_FREE, - * NXT_MEM_ZONE_PAGE_FRESH, and NXT_MEM_ZONE_PAGE_USED. - */ - uint16_t size; - - /* A number of free chunks of a chunked page. */ - uint16_t chunks; - - union { - /* A chunk bitmap if a number of chunks is lesser than 32. */ - uint8_t map[4]; - /* - * The count is a number of successive occupied pages in the first - * page. In the next occupied pages and in all free pages the count - * is zero, because a number of successive free pages is stored in - * free block size resided in beginning of the first free page. - */ - uint32_t count; - } u; - - /* Used for slot list of pages with free chunks. */ - nxt_mem_zone_page_t *next; - - /* - * Used to link of all pages including free, chunked and occupied - * pages to coalesce free pages. - */ - nxt_queue_link_t link; -}; - - -typedef struct { - uint32_t size; - uint32_t chunks; - uint32_t start; - uint32_t map_size; - nxt_mem_zone_page_t *pages; -} nxt_mem_zone_slot_t; - - -typedef struct { - NXT_RBTREE_NODE (node); - uint32_t size; -} nxt_mem_zone_free_block_t; - - -struct nxt_mem_zone_s { - nxt_thread_spinlock_t lock; - nxt_mem_zone_page_t *pages; - nxt_mem_zone_page_t sentinel_page; - nxt_rbtree_t free_pages; - - uint32_t page_size_shift; - uint32_t page_size_mask; - uint32_t max_chunk_size; - uint32_t small_bitmap_min_size; - - u_char *start; - u_char *end; - - nxt_mem_zone_slot_t slots[]; -}; - - -#define nxt_mem_zone_page_addr(zone, page) \ - (void *) (zone->start + ((page - zone->pages) << zone->page_size_shift)) - - -#define nxt_mem_zone_addr_page(zone, addr) \ - &zone->pages[((u_char *) addr - zone->start) >> zone->page_size_shift] - - -#define nxt_mem_zone_page_is_free(page) \ - (page->size < NXT_MEM_ZONE_PAGE_USED) - - -#define nxt_mem_zone_page_is_chunked(page) \ - (page->size >= 16) - - -#define nxt_mem_zone_page_bitmap(zone, slot) \ - (slot->size < zone->small_bitmap_min_size) - - -#define nxt_mem_zone_set_chunk_free(map, chunk) \ - map[chunk / 8] &= ~(0x80 >> (chunk & 7)) - - -#define nxt_mem_zone_chunk_is_free(map, chunk) \ - ((map[chunk / 8] & (0x80 >> (chunk & 7))) == 0) - - -#define nxt_mem_zone_fresh_junk(p, size) \ - nxt_memset((p), 0xA5, size) - - -#define nxt_mem_zone_free_junk(p, size) \ - nxt_memset((p), 0x5A, size) - - -static uint32_t nxt_mem_zone_pages(u_char *start, size_t zone_size, - nxt_uint_t page_size); -static void *nxt_mem_zone_slots_init(nxt_mem_zone_t *zone, - nxt_uint_t page_size); -static void nxt_mem_zone_slot_init(nxt_mem_zone_slot_t *slot, - nxt_uint_t page_size); -static intptr_t nxt_mem_zone_rbtree_compare(nxt_rbtree_node_t *node1, - nxt_rbtree_node_t *node2); -static void *nxt_mem_zone_alloc_small(nxt_mem_zone_t *zone, - nxt_mem_zone_slot_t *slot, size_t size); -static nxt_uint_t nxt_mem_zone_alloc_chunk(uint8_t *map, nxt_uint_t offset, - nxt_uint_t size); -static void *nxt_mem_zone_alloc_large(nxt_mem_zone_t *zone, size_t alignment, - size_t size); -static nxt_mem_zone_page_t *nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone, - size_t alignment, uint32_t pages); -static nxt_mem_zone_free_block_t * - nxt_mem_zone_find_free_block(nxt_mem_zone_t *zone, nxt_rbtree_node_t *node, - uint32_t alignment, uint32_t pages); -static const char *nxt_mem_zone_free_chunk(nxt_mem_zone_t *zone, - nxt_mem_zone_page_t *page, void *p); -static void nxt_mem_zone_free_pages(nxt_mem_zone_t *zone, - nxt_mem_zone_page_t *page, nxt_uint_t count); - - -static nxt_log_moderation_t nxt_mem_zone_log_moderation = { - NXT_LOG_ALERT, 2, "mem_zone_alloc() failed, not enough memory", - NXT_LOG_MODERATION -}; - - -nxt_mem_zone_t * -nxt_mem_zone_init(u_char *start, size_t zone_size, nxt_uint_t page_size) -{ - uint32_t pages; - nxt_uint_t n; - nxt_mem_zone_t *zone; - nxt_mem_zone_page_t *page; - nxt_mem_zone_free_block_t *block; - - if (nxt_slow_path((page_size & (page_size - 1)) != 0)) { - nxt_thread_log_alert("mem zone page size must be a power of 2"); - return NULL; - } - - pages = nxt_mem_zone_pages(start, zone_size, page_size); - if (pages == 0) { - return NULL; - } - - zone = (nxt_mem_zone_t *) start; - - /* The function returns address after all slots. */ - page = nxt_mem_zone_slots_init(zone, page_size); - - zone->pages = page; - - for (n = 0; n < pages; n++) { - page[n].size = NXT_MEM_ZONE_PAGE_FRESH; - } - - /* - * A special sentinel page entry marked as used does not correspond - * to a real page. The entry simplifies neighbour queue nodes check - * in nxt_mem_zone_free_pages(). - */ - zone->sentinel_page.size = NXT_MEM_ZONE_PAGE_USED; - nxt_queue_sentinel(&zone->sentinel_page.link); - nxt_queue_insert_after(&zone->sentinel_page.link, &page->link); - - /* rbtree of free pages. */ - - nxt_rbtree_init(&zone->free_pages, nxt_mem_zone_rbtree_compare); - - block = (nxt_mem_zone_free_block_t *) zone->start; - block->size = pages; - - nxt_rbtree_insert(&zone->free_pages, &block->node); - - return zone; -} - - -static uint32_t -nxt_mem_zone_pages(u_char *start, size_t zone_size, nxt_uint_t page_size) -{ - u_char *end; - size_t reserved; - nxt_uint_t n, pages, size, chunks, last; - nxt_mem_zone_t *zone; - - /* - * Find all maximum chunk sizes which zone page can be split on - * with minimum 16-byte step. - */ - last = page_size / 16; - n = 0; - size = 32; - - do { - chunks = page_size / size; - - if (last != chunks) { - last = chunks; - n++; - } - - size += 16; - - } while (chunks > 1); - - /* - * Find number of usable zone pages except zone bookkeeping data, - * slots, and pages entries. - */ - reserved = sizeof(nxt_mem_zone_t) + (n * sizeof(nxt_mem_zone_slot_t)); - - end = nxt_trunc_ptr(start + zone_size, page_size); - zone_size = end - start; - - pages = (zone_size - reserved) / (page_size + sizeof(nxt_mem_zone_page_t)); - - if (reserved > zone_size || pages == 0) { - nxt_thread_log_alert("mem zone size is too small: %uz", zone_size); - return 0; - } - - reserved += pages * sizeof(nxt_mem_zone_page_t); - nxt_memzero(start, reserved); - - zone = (nxt_mem_zone_t *) start; - - zone->start = nxt_align_ptr(start + reserved, page_size); - zone->end = end; - - nxt_thread_log_debug("mem zone pages: %uD, unused:%z", pages, - end - (zone->start + pages * page_size)); - - /* - * If a chunk size is lesser than zone->small_bitmap_min_size - * bytes, a page's chunk bitmap is larger than 32 bits and the - * bimap is placed at the start of the page. - */ - zone->small_bitmap_min_size = page_size / 32; - - zone->page_size_mask = page_size - 1; - zone->max_chunk_size = page_size / 2; - - n = zone->max_chunk_size; - - do { - zone->page_size_shift++; - n /= 2; - } while (n != 0); - - return (uint32_t) pages; -} - - -static void * -nxt_mem_zone_slots_init(nxt_mem_zone_t *zone, nxt_uint_t page_size) -{ - nxt_uint_t n, size, chunks; - nxt_mem_zone_slot_t *slot; - - slot = zone->slots; - - slot[0].chunks = page_size / 16; - slot[0].size = 16; - - n = 0; - size = 32; - - for ( ;; ) { - chunks = page_size / size; - - if (slot[n].chunks != chunks) { - - nxt_mem_zone_slot_init(&slot[n], page_size); - - nxt_thread_log_debug( - "mem zone size:%uD chunks:%uD start:%uD map:%uD", - slot[n].size, slot[n].chunks + 1, - slot[n].start, slot[n].map_size); - - n++; - - if (chunks == 1) { - return &slot[n]; - } - } - - slot[n].chunks = chunks; - slot[n].size = size; - size += 16; - } -} - - -static void -nxt_mem_zone_slot_init(nxt_mem_zone_slot_t *slot, nxt_uint_t page_size) -{ - /* - * Calculate number of bytes required to store a chunk bitmap - * and align it to 4 bytes. - */ - slot->map_size = nxt_align_size(((slot->chunks + 7) / 8), 4); - - /* If chunk size is not a multiple of zone page size, there - * is surplus space which can be used for the chunk's bitmap. - */ - slot->start = page_size - slot->chunks * slot->size; - - /* slot->chunks should be one less than actual number of chunks. */ - slot->chunks--; - - if (slot->map_size > 4) { - /* A page's chunks bitmap is placed at the start of the page. */ - - if (slot->start < slot->map_size) { - /* - * There is no surplus space or the space is too - * small for chunks bitmap, so use the first chunks. - */ - if (slot->size < slot->map_size) { - /* The first chunks are occupied by bitmap. */ - slot->chunks -= slot->map_size / slot->size; - slot->start = nxt_align_size(slot->map_size, 16); - - } else { - /* The first chunk is occupied by bitmap. */ - slot->chunks--; - slot->start = slot->size; - } - } - } -} - - -/* - * Round up to the next highest power of 2. The algorithm is - * described in "Bit Twiddling Hacks" by Sean Eron Anderson. - */ - -nxt_inline uint32_t -nxt_next_highest_power_of_two(uint32_t n) -{ - n--; - n |= n >> 1; - n |= n >> 2; - n |= n >> 4; - n |= n >> 8; - n |= n >> 16; - n++; - - return n; -} - - -static intptr_t -nxt_mem_zone_rbtree_compare(nxt_rbtree_node_t *node1, nxt_rbtree_node_t *node2) -{ - u_char *start1, *end1, *start2, *end2; - uint32_t n, size, size1, size2; - nxt_mem_zone_free_block_t *block1, *block2; - - block1 = (nxt_mem_zone_free_block_t *) node1; - block2 = (nxt_mem_zone_free_block_t *) node2; - - size1 = block1->size; - size2 = block2->size; - - /* - * This subtractions do not overflow if number of pages of a free - * block is below 2^31-1. This allows to use blocks up to 128G if - * a zone page size is just 64 bytes. - */ - n = size1 - size2; - - if (n != 0) { - return n; - } - - /* - * Sort equally sized blocks by their capability to allocate memory with - * alignment equal to the size rounded the previous higest power of 2. - */ - - /* Round the size to the previous higest power of two. */ - size = nxt_next_highest_power_of_two(size1) >> 1; - - /* Align the blocks' start and end to the rounded size. */ - start1 = nxt_align_ptr(block1, size); - end1 = nxt_trunc_ptr((u_char *) block1 + size1, size); - - start2 = nxt_align_ptr(block2, size); - end2 = nxt_trunc_ptr((u_char *) block2 + size2, size); - - return (end1 - start1) - (end2 - start2); -} - - -void * -nxt_mem_zone_zalloc(nxt_mem_zone_t *zone, size_t size) -{ - void *p; - - p = nxt_mem_zone_align(zone, 1, size); - - if (nxt_fast_path(p != NULL)) { - nxt_memzero(p, size); - } - - return p; -} - - -void * -nxt_mem_zone_align(nxt_mem_zone_t *zone, size_t alignment, size_t size) -{ - void *p; - nxt_mem_zone_slot_t *slot; - - if (nxt_slow_path((alignment - 1) & alignment) != 0) { - /* Alignment must be a power of 2. */ - return NULL; - } - - if (size <= zone->max_chunk_size && alignment <= zone->max_chunk_size) { - /* All chunks are aligned to 16. */ - - if (alignment > 16) { - /* - * Chunks which size is power of 2 are aligned to the size. - * So allocation size should be increased to the next highest - * power of two. This can waste memory, but a main consumer - * of aligned allocations is lvlhsh which anyway allocates - * memory with alignment equal to size. - */ - size = nxt_next_highest_power_of_two(size); - size = nxt_max(size, alignment); - } - - /* - * Find a zone slot with appropriate chunk size. - * This operation can be performed without holding lock. - */ - for (slot = zone->slots; slot->size < size; slot++) { /* void */ } - - nxt_thread_log_debug("mem zone alloc: @%uz:%uz chunk:%uD", - alignment, size, slot->size); - - nxt_thread_spin_lock(&zone->lock); - - p = nxt_mem_zone_alloc_small(zone, slot, size); - - } else { - - nxt_thread_log_debug("mem zone alloc: @%uz:%uz", alignment, size); - - nxt_thread_spin_lock(&zone->lock); - - p = nxt_mem_zone_alloc_large(zone, alignment, size); - } - - nxt_thread_spin_unlock(&zone->lock); - - if (nxt_fast_path(p != NULL)) { - nxt_thread_log_debug("mem zone alloc: %p", p); - - } else { - nxt_log_alert_moderate(&nxt_mem_zone_log_moderation, nxt_thread_log(), - "nxt_mem_zone_alloc(%uz, %uz) failed, not enough memory", - alignment, size); - } - - return p; -} - - -static void * -nxt_mem_zone_alloc_small(nxt_mem_zone_t *zone, nxt_mem_zone_slot_t *slot, - size_t size) -{ - u_char *p; - uint8_t *map; - nxt_mem_zone_page_t *page; - - page = slot->pages; - - if (nxt_fast_path(page != NULL)) { - - p = nxt_mem_zone_page_addr(zone, page); - - if (nxt_mem_zone_page_bitmap(zone, slot)) { - /* A page's chunks bitmap is placed at the start of the page. */ - map = p; - - } else { - map = page->u.map; - } - - p += nxt_mem_zone_alloc_chunk(map, slot->start, slot->size); - - page->chunks--; - - if (page->chunks == 0) { - /* - * Remove full page from the zone slot list of pages with - * free chunks. - */ - slot->pages = page->next; -#if (NXT_DEBUG) - page->next = NULL; -#endif - } - - return p; - } - - page = nxt_mem_zone_alloc_pages(zone, 1, 1); - - if (nxt_fast_path(page != NULL)) { - - slot->pages = page; - - page->size = slot->size; - /* slot->chunks are already one less. */ - page->chunks = slot->chunks; - page->u.count = 0; - page->next = NULL; - - p = nxt_mem_zone_page_addr(zone, page); - - if (nxt_mem_zone_page_bitmap(zone, slot)) { - /* A page's chunks bitmap is placed at the start of the page. */ - map = p; - nxt_memzero(map, slot->map_size); - - } else { - map = page->u.map; - } - - /* Mark the first chunk as busy. */ - map[0] = 0x80; - - return p + slot->start; - } - - return NULL; -} - - -static nxt_uint_t -nxt_mem_zone_alloc_chunk(uint8_t *map, nxt_uint_t offset, nxt_uint_t size) -{ - uint8_t mask; - nxt_uint_t n; - - n = 0; - - /* The page must have at least one free chunk. */ - - for ( ;; ) { - /* The bitmap is always aligned to uint32_t. */ - - if (*(uint32_t *) &map[n] != 0xFFFFFFFF) { - - do { - if (map[n] != 0xFF) { - - mask = 0x80; - - do { - if ((map[n] & mask) == 0) { - /* The free chunk is found. */ - map[n] |= mask; - return offset; - } - - offset += size; - mask >>= 1; - - } while (mask != 0); - - } else { - /* Fast-forward: all 8 chunks are occupied. */ - offset += size * 8; - } - - n++; - - } while (n % 4 != 0); - - } else { - /* Fast-forward: all 32 chunks are occupied. */ - offset += size * 32; - n += 4; - } - } -} - - -static void * -nxt_mem_zone_alloc_large(nxt_mem_zone_t *zone, size_t alignment, size_t size) -{ - uint32_t pages; - nxt_mem_zone_page_t *page; - - pages = (size + zone->page_size_mask) >> zone->page_size_shift; - - page = nxt_mem_zone_alloc_pages(zone, alignment, pages); - - if (nxt_fast_path(page != NULL)) { - return nxt_mem_zone_page_addr(zone, page); - } - - return NULL; -} - - -static nxt_mem_zone_page_t * -nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone, size_t alignment, uint32_t pages) -{ - u_char *p; - size_t prev_size; - uint32_t prev_pages, node_pages, next_pages; - nxt_uint_t n; - nxt_mem_zone_page_t *prev_page, *page, *next_page; - nxt_mem_zone_free_block_t *block, *next_block; - - block = nxt_mem_zone_find_free_block(zone, - nxt_rbtree_root(&zone->free_pages), - alignment, pages); - - if (nxt_slow_path(block == NULL)) { - return NULL; - } - - node_pages = block->size; - - nxt_rbtree_delete(&zone->free_pages, &block->node); - - p = nxt_align_ptr(block, alignment); - page = nxt_mem_zone_addr_page(zone, p); - - prev_size = p - (u_char *) block; - - if (prev_size != 0) { - prev_pages = prev_size >> zone->page_size_shift; - node_pages -= prev_pages; - - block->size = prev_pages; - nxt_rbtree_insert(&zone->free_pages, &block->node); - - prev_page = nxt_mem_zone_addr_page(zone, block); - nxt_queue_insert_after(&prev_page->link, &page->link); - } - - next_pages = node_pages - pages; - - if (next_pages != 0) { - next_page = &page[pages]; - next_block = nxt_mem_zone_page_addr(zone, next_page); - next_block->size = next_pages; - - nxt_rbtree_insert(&zone->free_pages, &next_block->node); - nxt_queue_insert_after(&page->link, &next_page->link); - } - - /* Go through pages after all rbtree operations to not trash CPU cache. */ - - page[0].u.count = pages; - - for (n = 0; n < pages; n++) { - - if (page[n].size == NXT_MEM_ZONE_PAGE_FRESH) { - nxt_mem_zone_fresh_junk(nxt_mem_zone_page_addr(zone, &page[n]), - zone->page_size_mask + 1); - } - - page[n].size = NXT_MEM_ZONE_PAGE_USED; - } - - return page; -} - - -/* - * Free blocks are sorted by size and then if the sizes are equal - * by aligned allocation capabilty. The former criterion is just - * comparison with a requested size and it can be used for iteractive - * search. The later criterion cannot be tested only by the requested - * size and alignment, so recursive in-order tree traversal is required - * to find a suitable free block. nxt_mem_zone_find_free_block() uses - * only recursive in-order tree traversal because anyway the slowest part - * of the algorithm are CPU cache misses. Besides the last tail recursive - * call may be optimized by compiler into iteractive search. - */ - -static nxt_mem_zone_free_block_t * -nxt_mem_zone_find_free_block(nxt_mem_zone_t *zone, nxt_rbtree_node_t *node, - uint32_t alignment, uint32_t pages) -{ - u_char *aligned, *end; - nxt_mem_zone_free_block_t *block, *free_block; - - if (node == nxt_rbtree_sentinel(&zone->free_pages)) { - return NULL; - } - - block = (nxt_mem_zone_free_block_t *) node; - - if (pages <= block->size) { - - free_block = nxt_mem_zone_find_free_block(zone, block->node.left, - alignment, pages); - if (free_block != NULL) { - return free_block; - } - - aligned = nxt_align_ptr(block, alignment); - - if (pages == block->size) { - if (aligned == (u_char *) block) { - /* Exact match. */ - return block; - } - - } else { /* pages < block->size */ - aligned += pages << zone->page_size_shift; - end = nxt_pointer_to(block, block->size << zone->page_size_shift); - - if (aligned <= end) { - return block; - } - } - } - - return nxt_mem_zone_find_free_block(zone, block->node.right, - alignment, pages); -} - - -void -nxt_mem_zone_free(nxt_mem_zone_t *zone, void *p) -{ - nxt_uint_t count; - const char *err; - nxt_mem_zone_page_t *page; - - nxt_thread_log_debug("mem zone free: %p", p); - - if (nxt_fast_path(zone->start <= (u_char *) p - && (u_char *) p < zone->end)) - { - page = nxt_mem_zone_addr_page(zone, p); - - nxt_thread_spin_lock(&zone->lock); - - if (nxt_mem_zone_page_is_chunked(page)) { - err = nxt_mem_zone_free_chunk(zone, page, p); - - } else if (nxt_slow_path(nxt_mem_zone_page_is_free(page))) { - err = "page is already free"; - - } else if (nxt_slow_path((uintptr_t) p & zone->page_size_mask) != 0) { - err = "invalid pointer to chunk"; - - } else { - count = page->u.count; - - if (nxt_fast_path(count != 0)) { - nxt_mem_zone_free_junk(p, count * zone->page_size_mask + 1); - nxt_mem_zone_free_pages(zone, page, count); - err = NULL; - - } else { - /* Not the first allocated page. */ - err = "pointer to wrong page"; - } - } - - nxt_thread_spin_unlock(&zone->lock); - - } else { - err = "pointer is out of zone"; - } - - if (nxt_slow_path(err != NULL)) { - nxt_thread_log_alert("nxt_mem_zone_free(%p): %s", p, err); - } -} - - -static const char * -nxt_mem_zone_free_chunk(nxt_mem_zone_t *zone, nxt_mem_zone_page_t *page, - void *p) -{ - u_char *map; - uint32_t size, offset, chunk; - nxt_mem_zone_page_t *pg, **ppg; - nxt_mem_zone_slot_t *slot; - - size = page->size; - - /* Find a zone slot with appropriate chunk size. */ - for (slot = zone->slots; slot->size < size; slot++) { /* void */ } - - offset = (uintptr_t) p & zone->page_size_mask; - offset -= slot->start; - - chunk = offset / size; - - if (nxt_slow_path(offset != chunk * size)) { - return "pointer to wrong chunk"; - } - - if (nxt_mem_zone_page_bitmap(zone, slot)) { - /* A page's chunks bitmap is placed at the start of the page. */ - map = (u_char *) ((uintptr_t) p & ~((uintptr_t) zone->page_size_mask)); - - } else { - map = page->u.map; - } - - if (nxt_mem_zone_chunk_is_free(map, chunk)) { - return "chunk is already free"; - } - - nxt_mem_zone_set_chunk_free(map, chunk); - - nxt_mem_zone_free_junk(p, page->size); - - if (page->chunks == 0) { - page->chunks = 1; - - /* Add the page to the head of slot list of pages with free chunks. */ - page->next = slot->pages; - slot->pages = page; - - } else if (page->chunks != slot->chunks) { - page->chunks++; - - } else { - - if (map != page->u.map) { - nxt_mem_zone_free_junk(map, slot->map_size); - } - - /* - * All chunks are free, remove the page from the slot list of pages - * with free chunks and add the page to the free pages tree. - */ - ppg = &slot->pages; - - for (pg = slot->pages; pg != NULL; pg = pg->next) { - - if (pg == page) { - *ppg = page->next; - break; - } - - ppg = &pg->next; - } - - nxt_mem_zone_free_pages(zone, page, 1); - } - - return NULL; -} - - -static void -nxt_mem_zone_free_pages(nxt_mem_zone_t *zone, nxt_mem_zone_page_t *page, - nxt_uint_t count) -{ - nxt_mem_zone_page_t *prev_page, *next_page; - nxt_mem_zone_free_block_t *block, *prev_block, *next_block; - - page->size = NXT_MEM_ZONE_PAGE_FREE; - page->chunks = 0; - page->u.count = 0; - page->next = NULL; - - nxt_memzero(&page[1], (count - 1) * sizeof(nxt_mem_zone_page_t)); - - next_page = nxt_queue_link_data(page->link.next, nxt_mem_zone_page_t, link); - - if (nxt_mem_zone_page_is_free(next_page)) { - - /* Coalesce with the next free pages. */ - - nxt_queue_remove(&next_page->link); - nxt_memzero(next_page, sizeof(nxt_mem_zone_page_t)); - - next_block = nxt_mem_zone_page_addr(zone, next_page); - count += next_block->size; - nxt_rbtree_delete(&zone->free_pages, &next_block->node); - } - - prev_page = nxt_queue_link_data(page->link.prev, nxt_mem_zone_page_t, link); - - if (nxt_mem_zone_page_is_free(prev_page)) { - - /* Coalesce with the previous free pages. */ - - nxt_queue_remove(&page->link); - - prev_block = nxt_mem_zone_page_addr(zone, prev_page); - count += prev_block->size; - nxt_rbtree_delete(&zone->free_pages, &prev_block->node); - - prev_block->size = count; - nxt_rbtree_insert(&zone->free_pages, &prev_block->node); - - return; - } - - block = nxt_mem_zone_page_addr(zone, page); - block->size = count; - nxt_rbtree_insert(&zone->free_pages, &block->node); -} diff --git a/src/nxt_mem_zone.h b/src/nxt_mem_zone.h deleted file mode 100644 index 89d73ca22..000000000 --- a/src/nxt_mem_zone.h +++ /dev/null @@ -1,28 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#ifndef _NXT_MEM_ZONE_H_INCLUDED_ -#define _NXT_MEM_ZONE_H_INCLUDED_ - - -typedef struct nxt_mem_zone_s nxt_mem_zone_t; - - -NXT_EXPORT nxt_mem_zone_t *nxt_mem_zone_init(u_char *start, size_t zone_size, - nxt_uint_t page_size); - -#define nxt_mem_zone_alloc(zone, size) \ - nxt_mem_zone_align((zone), 1, (size)) - -NXT_EXPORT void *nxt_mem_zone_align(nxt_mem_zone_t *zone, size_t alignment, - size_t size) - NXT_MALLOC_LIKE; -NXT_EXPORT void *nxt_mem_zone_zalloc(nxt_mem_zone_t *zone, size_t size) - NXT_MALLOC_LIKE; -NXT_EXPORT void nxt_mem_zone_free(nxt_mem_zone_t *zone, void *p); - - -#endif /* _NXT_MEM_ZONE_H_INCLUDED_ */ diff --git a/src/test/nxt_mem_zone_test.c b/src/test/nxt_mem_zone_test.c deleted file mode 100644 index faf0feee6..000000000 --- a/src/test/nxt_mem_zone_test.c +++ /dev/null @@ -1,74 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include -#include "nxt_tests.h" - - -nxt_int_t -nxt_mem_zone_test(nxt_thread_t *thr, nxt_uint_t runs, nxt_uint_t nblocks, - size_t max_size) -{ - void *start, **blocks; - size_t total, zone_size; - uint32_t size; - nxt_uint_t i, n; - nxt_mem_zone_t *zone; - const size_t page_size = 4096; - - nxt_thread_time_update(thr); - nxt_log_error(NXT_LOG_NOTICE, thr->log, - "mem zone test started, max:%uz", max_size); - - zone_size = (max_size + 1) * nblocks; - - start = nxt_memalign(page_size, zone_size); - if (start == NULL) { - return NXT_ERROR; - } - - zone = nxt_mem_zone_init(start, zone_size, page_size); - if (zone == NULL) { - return NXT_ERROR; - } - - blocks = nxt_malloc(nblocks * sizeof(void *)); - if (blocks == NULL) { - return NXT_ERROR; - } - - size = 0; - - for (i = 0; i < runs; i++) { - - total = 0; - - for (n = 0; n < nblocks; n++) { - size = nxt_murmur_hash2(&size, sizeof(uint32_t)); - - total += size & max_size; - blocks[n] = nxt_mem_zone_alloc(zone, size & max_size); - - if (blocks[n] == NULL) { - nxt_log_error(NXT_LOG_NOTICE, thr->log, - "mem zone test failed: %uz", total); - return NXT_ERROR; - } - } - - for (n = 0; n < nblocks; n++) { - nxt_mem_zone_free(zone, blocks[n]); - } - } - - nxt_free(blocks); - nxt_free(zone); - - nxt_thread_time_update(thr); - nxt_log_error(NXT_LOG_NOTICE, thr->log, "mem zone test passed"); - - return NXT_OK; -} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 55d1d5de0..a41ae2696 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -127,18 +127,6 @@ main(int argc, char **argv) return 1; } - if (nxt_mem_zone_test(thr, 100, 20000, 128 - 1) != NXT_OK) { - return 1; - } - - if (nxt_mem_zone_test(thr, 100, 10000, 4096 - 1) != NXT_OK) { - return 1; - } - - if (nxt_mem_zone_test(thr, 1000, 40, 64 * 1024 - 1) != NXT_OK) { - return 1; - } - if (nxt_lvlhsh_test(thr, 2, 1) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index d2a381163..6061308bd 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -54,8 +54,6 @@ void nxt_rbtree1_mb_delete(nxt_thread_t *thr); nxt_int_t nxt_mp_test(nxt_thread_t *thr, nxt_uint_t runs, nxt_uint_t nblocks, size_t max_size); nxt_int_t nxt_mp_get_align_test(nxt_thread_t *thr); -nxt_int_t nxt_mem_zone_test(nxt_thread_t *thr, nxt_uint_t runs, - nxt_uint_t nblocks, size_t max_size); nxt_int_t nxt_lvlhsh_test(nxt_thread_t *thr, nxt_uint_t n, nxt_bool_t use_pool); From fcd389f21dd68b9917fc605060247ca0f0f46377 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 02:23:53 +0200 Subject: [PATCH 177/391] ci(unitctl): build and test the musl, TLS-free state The "tls" Cargo feature is off for musl, and nothing in CI compiles that state. Both #[cfg(not(feature = "tls"))] blocks and the test that an "https://" control socket is refused could break and no run would say so. Add one job. It builds the musl target, which the GNUmakefile builds without default features, then runs the test suite on the host in the same feature state, so the feature-off branches are executed and not only compiled. Four assertions, each on an artifact rather than on a log line: The binary carries no aws_lc symbol. The symbol count is checked first, because a stripped binary would pass that test while proving nothing. Measured: 0 aws_lc symbols with the feature off, 1756 with it on. The binary has no PT_INTERP segment. A program that links a C library needs an interpreter to load it. readelf names the segment; "file" would say "static-pie linked", which is that tool's wording and moves with the runner image. No cc, cmake or aws-lc-sys is in the feature-off dependency graph. The runner has a C toolchain, so the job cannot show that a build works without one; it can show that nothing in the graph would call for it. The tree line count is asserted first, so an empty tree cannot pass. The https refusal test ran. A whole-suite run stays green if that test is deleted or marked #[ignore], and a name filter that matches nothing also reports "ok" -- with "0 passed". The step requires "1 passed". Last, a native build on a musl host takes the same flag by a different rule (HOST_TARGET), and no runner has a musl host. Ask make what it would run, in both directions: a rule that always passed the flag would break every gnu build. The job keeps no cache. The make recipe skips the build when the binary is already there, so a restored target directory would turn this gate into a no-op. The pull request path filter now lists this workflow file. Without that a pull request that only edits the workflow never runs it, so an error here would land unchecked. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/unitctl.yml | 96 +++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/.github/workflows/unitctl.yml b/.github/workflows/unitctl.yml index 7ca25cc03..752631a16 100644 --- a/.github/workflows/unitctl.yml +++ b/.github/workflows/unitctl.yml @@ -6,6 +6,9 @@ on: paths: - tools/unitctl/** - docs/unit-openapi.yaml + # A change to this file must run this file. Without it a pull request + # that only edits the workflow is never checked by it. + - .github/workflows/unitctl.yml push: branches: - master @@ -112,6 +115,99 @@ jobs: - name: Audit Cargo.lock run: cargo audit --deny warnings + musl: + # The only leg that compiles unitctl without the "tls" feature. The + # feature-off code -- the #[cfg(not(feature = "tls"))] blocks and the test + # that an "https://" control socket is refused -- is built nowhere else, so + # without this job it can break and no run would say so. + # + # The musl target is also the one shipped without TLS, because TLS is what + # pulls in aws-lc-sys and its C toolchain. + # + # What this job does not cover: every runner has a gnu host, so a musl + # *host* -- "make release" on Alpine -- is not built here. The make rule + # for that host is checked below instead, which is the part that can be + # checked without a musl runner. + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: tools/unitctl + steps: + - uses: actions/checkout@v5 + + - run: rustup update stable + - run: rustup target add x86_64-unknown-linux-musl + + # The build needs the generated OpenAPI client, which is produced by a + # Java tool. + - uses: actions/setup-java@v6 + with: + distribution: temurin + java-version: 21 + + # The GNUmakefile drops the default features for a musl target, so this + # is the feature-off build. + - name: Make unitctl (x86_64-unknown-linux-musl) + run: make x86_64-unknown-linux-musl + + # Assert on the binary, not on the build log. Without TLS there is no + # aws-lc-sys, so the binary carries no aws_lc symbol. The symbol count is + # checked first, because a stripped binary would make the aws_lc test pass + # while proving nothing. + # + # A program that links a C library needs an interpreter to load it, so the + # absence of a PT_INTERP segment is what says none was linked. readelf + # names the segment; "file" would say "static-pie linked", which is that + # tool's wording and changes with the runner image. + - name: Check the binary carries no TLS stack + run: | + bin=target/x86_64-unknown-linux-musl/release/unitctl + file "$bin" + test "$(nm "$bin" | wc -l)" -gt 1000 + ! nm "$bin" | grep -q aws_lc + ! readelf -lW "$bin" | grep -q INTERP + + # The claim behind the musl build is that TLS is the only thing needing a + # C toolchain. This runner has cc and cmake, so the job cannot show that + # a build works without them. What it can show is that no crate which + # would use them is in the feature-off graph. The line count guards + # against an empty tree passing the grep. + - name: Check no C-building crate is in the graph + run: | + tree=$(cargo tree --no-default-features \ + --target x86_64-unknown-linux-musl --edges normal,build) + test "$(echo "$tree" | wc -l)" -gt 50 + ! echo "$tree" | grep -E '(^|[^a-z-])(cc|cmake|aws-lc-sys) v' + + # Run the host tests in the same feature state, so the feature-off + # branches are executed and not merely compiled. + - name: Test without the tls feature + run: cargo test --no-default-features + + # The step above stays green if the refusal test is deleted or marked + # #[ignore]. Name the test and require that one test ran: a filter that + # matches nothing also reports "ok", with "0 passed". + - name: Check the https refusal test ran + run: | + cargo test --no-default-features -p unit-client-rs \ + control_socket_address::tests::refuses_https_without_the_tls_feature \ + -- --exact | tee result.log + grep -q 'test result: ok. 1 passed' result.log + rm result.log + + # A native build on a musl host takes the same feature flag, by a + # different rule (GNUmakefile HOST_TARGET). No runner has a musl host, + # so ask make what it would run. Both directions are checked: a rule + # that always passed the flag would break every gnu build. + - name: Check the native musl build drops the feature too + run: | + make -n release HOST_TARGET=x86_64-unknown-linux-musl \ + | grep -q -- '--no-default-features' + ! make -n release HOST_TARGET=x86_64-unknown-linux-gnu \ + | grep -q -- '--no-default-features' + build: # The weekly trigger exists for the audit job alone. if: github.event_name != 'schedule' From 11a0bcdf854ee469dbdbfff9510dc53b7cb1fc1b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 04:19:16 +0200 Subject: [PATCH 178/391] http: weaken the static ETag inside the mtime second Both validators Unit generates for a static file are derived from the whole-second mtime and the size, so a rewrite to the same size during the second the file was last written changes neither. RFC 9110 Sect. 8.8.1 calls a validator strong only when the representation cannot change again without it changing too, which ours cannot promise while the clock is still inside that second: another write can still land there. Mark the tag weak for as long as that is true, and strong once the second has passed, since no later write can reproduce the mtime. Apache weakens on the same condition, in ap_make_etag(). Only a "W/" prefix is added -- the tag's format does not change, so nothing already in a cache is invalidated, and both validators are covered rather than just the tag. The weakness has to be honoured, not just printed. It travels beside the tag as a flag while "etag" keeps holding the opaque tag, because the comparison works on what the client sends back: leaving "W/" in the string would make every If-None-Match miss. Under Sect. 8.8.3.2 a strong comparison needs both sides strong, so If-Match and the tag form of If-Range now fail inside the window, and the date form of If-Range fails with them -- Last-Modified is the same second-granularity validator. Getting that last one wrong splices bytes from two versions into one file, which is worse than the stale copy a bad conditional GET leaves. A strong comparison against "*" is unaffected: "*" asks whether a representation exists, not whether a validator matches. The test covers the window through a file dated one second ahead, which puts the request inside it without racing the clock, then waits the second out and finds the same file validating strongly with the same tag. Two of its cases send the opaque tag with no "W/" on it: that is the only way to observe Unit's own weakness rather than the client's, and it is what a cache holding the tag from before a same-second rewrite sends. Existing tests wrote a file and fetched it in the same second, so they sat inside the window by accident. They now backdate their fixtures, which is also what a deployed file looks like; the one test that dated a file into the future now dates it into the past, a future mtime being inside the window for the same reason. 54 static tests pass, 94 across the static, compression and response-header suites. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015j5NXxjo8hdYc99A8Db7tG --- src/nxt_http_static.c | 114 +++++++++++++++++++++++++++++++++--------- test/test_static.py | 100 +++++++++++++++++++++++++++++++++++- 2 files changed, 188 insertions(+), 26 deletions(-) diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 6ddc6f6ff..dbe2e81fe 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -168,12 +168,12 @@ static void nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, static void nxt_http_static_next(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx, nxt_http_status_t status); static nxt_http_status_t nxt_http_static_preconditions(nxt_http_request_t *r, - nxt_str_t *etag, nxt_time_t mtime); + nxt_str_t *etag, nxt_bool_t weak, nxt_time_t mtime); static nxt_bool_t nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, - nxt_bool_t strong); + nxt_bool_t own_weak, nxt_bool_t strong); static nxt_http_status_t nxt_http_static_range(nxt_http_request_t *r, - nxt_str_t *etag, nxt_time_t mtime, nxt_off_t size, nxt_off_t *start, - nxt_off_t *end); + nxt_str_t *etag, nxt_bool_t weak, nxt_time_t mtime, nxt_off_t size, + nxt_off_t *start, nxt_off_t *end); #if (NXT_HAVE_OPENAT2) static u_char *nxt_http_static_chroot_match(u_char *chr, u_char *shr); #endif @@ -488,7 +488,7 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx) { size_t length, encode; - u_char *p, *fname; + u_char *p, *end, *fname; struct tm tm; nxt_buf_t *fb; nxt_int_t ret; @@ -497,7 +497,7 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_file_t *f, file; nxt_file_info_t fi; nxt_off_t range_start, range_end; - nxt_bool_t is_range; + nxt_bool_t is_range, etag_weak; nxt_http_status_t rstatus; nxt_http_field_t *field; nxt_http_status_t status, pcond; @@ -723,7 +723,29 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_http_field_name_set(field, "ETag"); - length = NXT_TIME_T_HEXLEN + NXT_OFF_T_HEXLEN + 3; + /* + * RFC 9110 Sect. 8.8.1: a validator is strong only when the + * representation cannot change again without the validator changing + * with it. Both of ours are derived from the whole-second mtime and + * the size, so a rewrite to the same size during the second the file + * was last written is invisible to both. While the clock is still + * inside that second another write can still land there, so the tag + * cannot be promised strong; once the second has passed, no later + * write can reproduce this mtime and the tag is strong for good. + * + * Apache weakens on the same condition (server/util_etag.c, in + * ap_make_etag()). The tag's format does not change, so nothing + * already in a cache is invalidated by this. + * + * nxt_thread_time() is the cached per-thread clock. If it lags, it + * reports the request as still inside the second and the tag is + * weakened when it need not have been -- the safe direction. + */ + + etag_weak = ((nxt_time_t) nxt_thread_time(task->thread) + <= (nxt_time_t) nxt_file_mtime(&fi)); + + length = nxt_length("W/") + NXT_TIME_T_HEXLEN + NXT_OFF_T_HEXLEN + 3; p = nxt_mp_nget(r->mem_pool, length); if (nxt_slow_path(p == NULL)) { @@ -731,19 +753,33 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, } field->value = p; + + if (etag_weak) { + *p++ = 'W'; + *p++ = '/'; + } /* * nxt_file_mtime() yields a native time_t, which need not be * nxt_time_t: on QNX it is a 32-bit unsigned type against a 64-bit * nxt_time_t. "%T" reads an nxt_time_t from the argument list, so * the value has to be converted before it is passed, not after. */ - field->value_length = nxt_sprintf(p, p + length, "\"%xT-%xO\"", + end = field->value + length; + + field->value_length = nxt_sprintf(p, end, "\"%xT-%xO\"", (nxt_time_t) nxt_file_mtime(&fi), nxt_file_size(&fi)) - - p; + - field->value; + + /* + * The comparison functions work on the opaque tag, so "etag" skips + * the prefix; weakness travels beside it as a flag rather than in + * the string. Leaving "W/" in here would make every If-None-Match + * miss, since the client sends back the opaque tag it was given. + */ - etag.start = field->value; - etag.length = field->value_length; + etag.start = p; + etag.length = field->value_length - (p - field->value); if (exten.start == NULL) { nxt_http_static_extract_extension(shr, &exten); @@ -792,7 +828,8 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } - pcond = nxt_http_static_preconditions(r, &etag, nxt_file_mtime(&fi)); + pcond = nxt_http_static_preconditions(r, &etag, etag_weak, + nxt_file_mtime(&fi)); if (pcond != NXT_HTTP_OK) { nxt_file_close(task, f); @@ -839,7 +876,8 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, * reading of a sentence about methods that do not define ranges. */ - rstatus = nxt_http_static_range(r, &etag, nxt_file_mtime(&fi), + rstatus = nxt_http_static_range(r, &etag, etag_weak, + nxt_file_mtime(&fi), nxt_file_size(&fi), &range_start, &range_end); @@ -1081,7 +1119,7 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, static nxt_http_status_t nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, - nxt_time_t mtime) + nxt_bool_t weak, nxt_time_t mtime) { nxt_str_t value; nxt_time_t date; @@ -1148,7 +1186,8 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, /* Sect. 13.1.1: If-Match compares strongly. */ - im_match = nxt_http_static_etag_match(&value, etag, 1); + im_match = nxt_http_static_etag_match(&value, etag, weak, + 1); } } @@ -1164,7 +1203,8 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, value.start = f->value; value.length = f->value_length; - inm_match = nxt_http_static_etag_match(&value, etag, 0); + inm_match = nxt_http_static_etag_match(&value, etag, weak, + 0); } } @@ -1236,17 +1276,21 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, /* * Matches an entity-tag against an If-Match or If-None-Match list. * - * Unit's own tags are always strong, so the comparison functions differ only - * in how they treat a weak tag on the client's side: under the strong - * function (Sect. 8.8.3.2) a W/-prefixed tag never matches, under the weak - * one the prefix is stripped and the opaque tags are compared. + * "etag" is the opaque tag without any "W/"; "own_weak" says whether the tag + * Unit generated for this representation is weak, which it is while the + * request falls inside the second the file was last written. + * + * Sect. 8.8.3.2: the strong function matches only when both tags are strong, + * so a weak tag on either side fails it -- the client's, or our own. The + * weak function strips the prefix and compares the opaque tags. * * "*" matches any existing representation, but only as the entire field * value -- the grammar is "*" / #entity-tag, so it is not a list member. */ static nxt_bool_t -nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, nxt_bool_t strong) +nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, + nxt_bool_t own_weak, nxt_bool_t strong) { u_char *p, *end, *start; nxt_bool_t weak; @@ -1263,6 +1307,18 @@ nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, nxt_bool_t strong) return 1; } + if (own_weak && strong) { + /* + * A strong comparison needs both tags strong (Sect. 8.8.3.2), and + * ours is not, so no entity-tag in the list can match. This is + * below the "*" test on purpose: "*" asks whether a representation + * exists at all, not whether a validator matches, so weakness does + * not bear on it. + */ + + return 0; + } + while (p < end) { while (p < end && (*p == ' ' || *p == '\t' || *p == ',')) { @@ -1391,7 +1447,8 @@ nxt_http_static_range_number(u_char **p, u_char *end) static nxt_http_status_t nxt_http_static_range(nxt_http_request_t *r, nxt_str_t *etag, - nxt_time_t mtime, nxt_off_t size, nxt_off_t *start, nxt_off_t *end) + nxt_bool_t weak, nxt_time_t mtime, nxt_off_t size, nxt_off_t *start, + nxt_off_t *end) { u_char *p, *last; nxt_off_t a, b, suffix; @@ -1453,14 +1510,21 @@ nxt_http_static_range(nxt_http_request_t *r, nxt_str_t *etag, { /* An entity-tag: Sect. 13.1.5 requires the strong comparison. */ - match = nxt_http_static_etag_match(&value, etag, 1); + match = nxt_http_static_etag_match(&value, etag, weak, 1); } else { date = nxt_time_parse(value.start, value.length); - /* Sect. 13.1.5: an exact match against the last modification. */ + /* + * Sect. 13.1.5: an exact match against the last modification, + * and only while that date is a strong validator. Inside the + * second the file was written it is not: the client would splice + * a range from one version onto a copy of another, and unlike a + * bad conditional GET that produces a corrupt file rather than a + * stale one. Refusing the If-Range costs a full response. + */ - match = (date != (nxt_time_t) -1 && date == mtime); + match = (!weak && date != (nxt_time_t) -1 && date == mtime); } if (!match) { diff --git a/test/test_static.py b/test/test_static.py index 117e628a6..010abcd04 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -13,6 +13,24 @@ client = ApplicationProto() +def age_file(path, seconds=2): + """ + Backdates a file so its validators are strong. + + Both validators are derived from the whole-second mtime and the size, so + Unit marks them weak while the clock is still inside the second the file + was written -- another write could land in that second and change neither. + A test that writes a file and fetches it immediately is inside that window, + where a strong comparison (If-Match, If-Range) cannot match by design. + + Most tests here are about something else, so they get a file that looks + like a deployed one rather than one written microseconds ago. The window + itself is covered by test_static_conditional_etag_weak_within_mtime_second. + """ + when = os.stat(path).st_mtime - seconds + os.utime(path, (when, when)) + + @pytest.fixture(autouse=True) def setup_method_fixture(temp_dir): assets_dir = f'{temp_dir}/assets' @@ -23,6 +41,9 @@ def setup_method_fixture(temp_dir): Path(f'{assets_dir}/log.log').write_text('[debug]', encoding='utf-8') Path(f'{assets_dir}/dir/file').write_text('blah', encoding='utf-8') + for f in Path(assets_dir).rglob('*'): + age_file(f) + assert 'success' in client.conf( { "listeners": {"*:8080": {"pass": "routes"}}, @@ -1637,7 +1658,11 @@ def test_static_range_file_changes_between_requests(temp_dir): last_modified = r['headers']['Last-Modified'] path.write_text('abcdefghijklmnop', encoding='utf-8') - new_mtime = int(time.time()) + 5 + + # A distinct mtime, and in the past: a file dated in the future is inside + # the weak window too, since the clock has not left that second yet, and + # the tail of this test needs the rewritten file to validate strongly. + new_mtime = int(time.time()) - 5 os.utime(path, (new_mtime, new_mtime)) r = range_get(Range='bytes=10-', **{'If-Range': etag}) @@ -1774,3 +1799,76 @@ def test_static_range_numeric_overflow(): # Negative-looking inputs are simply not the grammar. for value in ['bytes=-5-9', 'bytes=-1-', 'bytes=5--9', 'bytes=-0-']: _check_range_honest(value, range_get(Range=value)) + + +def test_static_conditional_etag_weak_within_mtime_second(temp_dir): + # RFC 9110 Sect. 8.8.1: a validator is strong only when the + # representation cannot change again without the validator changing too. + # Both of Unit's are derived from the whole-second mtime and the size, so + # a rewrite to the same size inside the second the file was written is + # invisible to both. While the clock is still inside that second the tag + # is marked weak; once the second has passed it is strong for good. + path = Path(f'{temp_dir}/assets/index.html') + + # Dating the file one second ahead puts the request inside the window + # without racing the clock: a file the clock has not reached yet can + # still acquire that mtime. Waiting the second out then makes the same + # file strong, with the same mtime and the same size. + mtime = int(time.time()) + 1 + os.utime(path, (mtime, mtime)) + + weak = client.get(url='/index.html')['headers']['ETag'] + assert weak.startswith('W/'), 'weak inside the mtime second' + + # Weakness costs nothing else: a client caching on it still revalidates. + assert range_get(**{'If-None-Match': weak})['status'] == 304, 'weak 304' + + # A strong comparison cannot match a weak validator (Sect. 8.8.3.2), so + # If-Match fails and If-Range declines to serve a partial response. + assert range_get(**{'If-Match': weak})['status'] == 412, 'weak If-Match' + + resp = range_get(Range='bytes=0-4', **{'If-Range': weak}) + assert resp['status'] == 200, 'weak If-Range serves the whole file' + assert resp['body'] == '0123456789', 'and all of it' + + # A client that sends the opaque tag back without the "W/" is the case + # that tests Unit's own weakness rather than the client's: the list entry + # is strong, so only our side can fail the comparison. A cache holding + # the strong tag from before a same-second rewrite sends exactly this. + bare = weak[len('W/') :] + assert range_get(**{'If-Match': bare})['status'] == 412, 'bare If-Match' + + resp = range_get(Range='bytes=0-4', **{'If-Range': bare}) + assert resp['status'] == 200, 'bare weak If-Range serves the whole file' + + # Sect. 13.1.3 compares If-None-Match weakly, so the bare tag still hits. + assert range_get(**{'If-None-Match': bare})['status'] == 304, 'bare 304' + + # The date form of If-Range is the same validator and just as weak here. + # Getting this wrong splices two versions together in the client's file, + # which is worse than the stale copy a bad conditional GET leaves. + last_modified = client.get(url='/index.html')['headers']['Last-Modified'] + resp = range_get(Range='bytes=0-4', **{'If-Range': last_modified}) + assert resp['status'] == 200, 'weak If-Range date serves the whole file' + + # "*" asks whether a representation exists at all, not whether a + # validator matches, so the weakness does not bear on it. + assert range_get(**{'If-Match': '*'})['status'] == 200, 'If-Match *' + + # Wait for the clock to leave the second the file is dated to. That + # second spans [mtime, mtime + 1), so the wait ends at mtime + 1, not at + # mtime. Nothing about the file changes here, only the guarantee Unit + # can make about it. + while time.time() < mtime + 1: + time.sleep(0.05) + + strong = client.get(url='/index.html')['headers']['ETag'] + assert not strong.startswith('W/'), 'strong once the second has passed' + + # Only a prefix separates them: the tag's format does not change, so + # nothing already held in a cache is invalidated by the weakening. + assert weak == f'W/{strong}', 'same tag, weak prefix only' + + # And the strong comparisons work again on the very same bytes. + assert range_get(**{'If-Match': strong})['status'] == 200, 'strong match' + assert range_get(Range='bytes=0-4', **{'If-Range': strong})['status'] == 206 From 2edd375f79d2cf0b3e58387af5b38c49073b53bb Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 04:20:31 +0200 Subject: [PATCH 179/391] changelog: weak validators inside the mtime second Both files, same diff. Appended to the end of the unreleased 1.36.2 stanza; "ver=1.36.2" still counts 2 in docs/changes.xml and the XML parses. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015j5NXxjo8hdYc99A8Db7tG --- CHANGES | 11 +++++++++++ docs/changes.xml | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGES b/CHANGES index afecf2f6f..17bbb335d 100644 --- a/CHANGES +++ b/CHANGES @@ -237,6 +237,17 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 same conclusion and deleted the file as part of an unrelated cleanup. + *) Change: the entity-tag and last-modified date of a static file are + now weak while the request falls inside the second the file was + last written. Both are derived from the whole-second modification + time and the size, so a rewrite to the same size during that second + changes neither, and RFC 9110 Sect. 8.8.1 reserves a strong + validator for one that cannot miss a change. "If-Match" and + "If-Range" stop matching inside that window, so a range request + there is answered with the full 200 rather than a slice keyed to a + validator Unit cannot vouch for. The tag keeps its format, so + nothing already held in a cache is invalidated. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index 0b381c4f8..37e3875c0 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -315,6 +315,16 @@ deleted the file as part of an unrelated cleanup. + + +the entity-tag and last-modified date of a static file are now weak while +the request falls inside the second the file was last written, because a +rewrite to the same size during that second changes neither validator. +If-Match and If-Range stop matching in that window; the tag keeps its +format, so nothing already cached is invalidated. + + + From 49c8a5446a6ca5ee6cfedeb502ce3b251aeef7a3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 04:46:35 +0200 Subject: [PATCH 180/391] http: test the boundary the weak ETag turns on, and fix two claims Review findings, all four confirmed before acting on them. The test did not observe the condition it exists for. Dating the file one second ahead put every "weak" request in the "now < mtime" branch, so the "now == mtime" case -- the hazard itself, a rewrite landing in the second the file was written -- went untested: a build with "<" in place of "<=" passed the whole test. It now fails on the first assertion, checked by mutating the operator and rebuilding. It was also flaky, and not theoretically: it was seen to fail at the boundary. Unit reads a coarse cached clock, so Python's clock leads it by up to a jiffy and cannot say which second Unit is in. Both halves now use Unit's own "Date", which comes from that same cached second and is stamped after the validator is chosen: the test waits for Unit to report the second the file is dated to, and afterwards waits for the tag to harden and then checks it did not harden before that second ended. Eight consecutive runs pass. CHANGES said a range request in the window is answered with the full 200. Only one conditioned on If-Range is; a plain Range still gets its 206, now asserted so the claim cannot drift again. If-Match: * is called out as unaffected for the same reason. The Apache citation named the wrong file. The condition is in modules/http/http_etag.c, ap_make_etag_ex(), and it is a sliding second against a microsecond mtime rather than the calendar second a whole-second mtime gives us -- close, but not "the same condition". Also noted in the comment: the reasoning holds only where the clock that stamped the file is the one Unit reads. On a remote filesystem the mtime comes from the server, and a server clock that trails this one can still write into a second Unit has called strong. 94 passed, 3 skipped across the static, compression and response-header suites. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015j5NXxjo8hdYc99A8Db7tG --- CHANGES | 9 +++-- docs/changes.xml | 6 ++-- src/nxt_http_static.c | 12 +++++-- test/test_static.py | 82 +++++++++++++++++++++++++++++++++++-------- 4 files changed, 87 insertions(+), 22 deletions(-) diff --git a/CHANGES b/CHANGES index 17bbb335d..03258b3e4 100644 --- a/CHANGES +++ b/CHANGES @@ -244,9 +244,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 changes neither, and RFC 9110 Sect. 8.8.1 reserves a strong validator for one that cannot miss a change. "If-Match" and "If-Range" stop matching inside that window, so a range request - there is answered with the full 200 rather than a slice keyed to a - validator Unit cannot vouch for. The tag keeps its format, so - nothing already held in a cache is invalidated. + conditioned on "If-Range" is answered with the full 200 rather than + a slice keyed to a validator Unit cannot vouch for; an unconditional + "Range" is unaffected. "If-Match: *" is unaffected too, since "*" + asks whether a representation exists, not whether a validator + matches. The tag keeps its format, so nothing already held in a + cache is invalidated. Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index 37e3875c0..c78116730 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -320,8 +320,10 @@ deleted the file as part of an unrelated cleanup. the entity-tag and last-modified date of a static file are now weak while the request falls inside the second the file was last written, because a rewrite to the same size during that second changes neither validator. -If-Match and If-Range stop matching in that window; the tag keeps its -format, so nothing already cached is invalidated. +If-Match and the If-Range forms stop matching in that window, so a range +conditioned on If-Range falls back to the full response; an unconditional +Range is unaffected. The tag keeps its format, so nothing already cached is +invalidated. diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index dbe2e81fe..506200237 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -733,10 +733,18 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, * cannot be promised strong; once the second has passed, no later * write can reproduce this mtime and the tag is strong for good. * - * Apache weakens on the same condition (server/util_etag.c, in - * ap_make_etag()). The tag's format does not change, so nothing + * Apache weakens on the same hazard (modules/http/http_etag.c, in + * ap_make_etag_ex()), though on a sliding second against a + * microsecond mtime rather than the calendar second a whole-second + * mtime gives us. The tag's format does not change, so nothing * already in a cache is invalidated by this. * + * This reasons about the clock that stamped the file, so it holds + * only where that is the clock Unit reads. On a remote filesystem + * the mtime comes from the server: if that clock trails this one, + * Unit can call a tag strong while the server can still write into + * the second it names. Apache carries the same caveat. + * * nxt_thread_time() is the cached per-thread clock. If it lags, it * reports the request as still inside the second and the tag is * weakened when it need not have been -- the safe direction. diff --git a/test/test_static.py b/test/test_static.py index 010abcd04..c681b0bda 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -154,6 +154,36 @@ def test_static_accept_ranges(): assert resp['headers']['Accept-Ranges'] == 'bytes', 'Accept-Ranges on 200' +def unit_second(resp): + """ + The second Unit believes it is in, taken from the response it just sent. + + Unit stamps "Date" from the same cached coarse clock that decides whether + a validator is weak, and stamps it after the validator is chosen, so it + is the only clock a test can compare against without racing. + """ + return int(parsedate_to_datetime(resp['headers']['Date']).timestamp()) + + +def wait_for_unit_second(second, timeout=5): + """ + Returns a response Unit sent during exactly `second`. + + Used to put a request inside the second a file is dated to, which is + where both validators are weak. The file is dated far enough ahead that + the whole of that second is available once it arrives. + """ + deadline = time.time() + timeout + + while time.time() < deadline: + resp = client.get(url='/index.html') + + if unit_second(resp) == second: + return resp + + pytest.fail(f'Unit never reported second {second}') + + def range_get(**headers): return client.get( url='/index.html', @@ -1810,15 +1840,23 @@ def test_static_conditional_etag_weak_within_mtime_second(temp_dir): # is marked weak; once the second has passed it is strong for good. path = Path(f'{temp_dir}/assets/index.html') - # Dating the file one second ahead puts the request inside the window - # without racing the clock: a file the clock has not reached yet can - # still acquire that mtime. Waiting the second out then makes the same - # file strong, with the same mtime and the same size. - mtime = int(time.time()) + 1 + # Unit reads a coarse cached clock, so Python's clock cannot say which + # second Unit thinks it is in -- it leads Unit's by up to a jiffy, which + # made an earlier version of this test fail intermittently right at the + # boundary. Unit's own "Date" is rendered from that same cached second, + # so it is the oracle used throughout here. + # + # The file is dated two seconds ahead and the test then waits for Unit to + # report that exact second. That lands the request in the "now == mtime" + # case, which is the hazard itself: a rewrite landing in the second the + # file was written. Entering only the "now < mtime" case would leave the + # boundary untested -- a build with "<" in place of "<=" passes that. + mtime = int(time.time()) + 2 os.utime(path, (mtime, mtime)) - weak = client.get(url='/index.html')['headers']['ETag'] - assert weak.startswith('W/'), 'weak inside the mtime second' + resp = wait_for_unit_second(mtime) + weak = resp['headers']['ETag'] + assert weak.startswith('W/'), 'weak in the second the file is dated to' # Weakness costs nothing else: a client caching on it still revalidates. assert range_get(**{'If-None-Match': weak})['status'] == 304, 'weak 304' @@ -1855,15 +1893,29 @@ def test_static_conditional_etag_weak_within_mtime_second(temp_dir): # validator matches, so the weakness does not bear on it. assert range_get(**{'If-Match': '*'})['status'] == 200, 'If-Match *' - # Wait for the clock to leave the second the file is dated to. That - # second spans [mtime, mtime + 1), so the wait ends at mtime + 1, not at - # mtime. Nothing about the file changes here, only the guarantee Unit - # can make about it. - while time.time() < mtime + 1: - time.sleep(0.05) + # An unconditional Range is untouched. Only a range conditioned on a + # validator has anything to be wrong about, so weakening must not cost + # every range request in the window a partial response. + resp = range_get(Range='bytes=0-4') + assert resp['status'] == 206, 'plain Range still partial in the window' + assert resp['body'] == '01234', 'and the right slice' + + # Wait for the tag to go strong, then check it did not go strong early. + # Asserting on Unit's own "Date" rather than on a sleep is what keeps + # this off the boundary race: the tag may only harden once Unit has left + # the second the file is dated to. + deadline = time.time() + 5 + + while True: + resp = client.get(url='/index.html') + strong = resp['headers']['ETag'] + + if not strong.startswith('W/'): + break + + assert time.time() < deadline, 'tag never hardened' - strong = client.get(url='/index.html')['headers']['ETag'] - assert not strong.startswith('W/'), 'strong once the second has passed' + assert unit_second(resp) > mtime, 'hardened only after the second passed' # Only a prefix separates them: the tag's format does not change, so # nothing already held in a cache is invalidated by the weakening. From b7efd13ce656f407849e59b15af269b95f42a2cf Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 02:37:37 +0200 Subject: [PATCH 181/391] unitctl: send a JSON configuration file as it was written The PUT path read the file into a serde_json::Map and serialized that map back out. The map keeps one member per name, so two members with the same name became one before the server could see them, and every number was re-spelled by the serializer. Unit refuses a duplicate member itself (src/nxt_conf.c:1616) and points at the line and column of the file, so the check was not only lost, it was lost silently: the request succeeded and the configuration that got applied was not the one in the file. Send the bytes instead. JSON5 is still parsed and re-serialized, because it is not JSON and the server cannot read it. hjson and YAML are still refused by name. Two things follow from this: A file that is not valid UTF-8 is no longer refused here. Unit refuses it and names the member (src/nxt_conf_validation.c:1840), which is a better message than the parse error unitctl gave. "edit" now reads its temporary file as JSON, not JSON5, so a comment written in the editor is a syntax error. The file holds the JSON Unit returned, and what the editor leaves is what gets sent. Measured against a live unitd, same input file, old binary and new: old: 200, "Reconfiguration done.", first member dropped new: 400, "Duplicate object member.", line 3, column 3 Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/unitctl/src/cmd/edit.rs | 10 +- tools/unitctl/unitctl/src/cmd/execute.rs | 4 +- tools/unitctl/unitctl/src/cmd/import.rs | 8 +- tools/unitctl/unitctl/src/inputfile.rs | 157 +++++++++++++++-------- tools/unitctl/unitctl/src/known_size.rs | 8 +- tools/unitctl/unitctl/src/requests.rs | 31 ++--- 6 files changed, 135 insertions(+), 83 deletions(-) diff --git a/tools/unitctl/unitctl/src/cmd/edit.rs b/tools/unitctl/unitctl/src/cmd/edit.rs index 60c3f9493..7ae946865 100644 --- a/tools/unitctl/unitctl/src/cmd/edit.rs +++ b/tools/unitctl/unitctl/src/cmd/edit.rs @@ -1,5 +1,5 @@ use crate::inputfile::{InputFile, InputFormat}; -use crate::requests::{send_and_validate_config_deserialize_response, send_empty_body_read_bytes}; +use crate::requests::{send_config_deserialize_response, send_empty_body_read_bytes}; use crate::unitctl::UnitCtl; use crate::unitctl_error::ControlSocketErrorKind; use crate::{wait, OutputFormat, UnitctlError}; @@ -81,7 +81,11 @@ pub(crate) async fn cmd(cli: &UnitCtl, output_format: OutputFormat) -> Result<() let temp_file_path = temp_file.path(); let before_edit_mod_time = temp_file_path.metadata().ok().map(|m| m.modified().ok()); - let inputfile = InputFile::FileWithFormat(temp_file_path.into(), InputFormat::Json5); + // The temporary file holds what Unit returned, pretty-printed above, and + // whatever the editor leaves in it is sent as it is. Reading it as JSON5 + // would let a comment be written here, but it would also parse and + // re-serialize the file, which is what this path no longer does. + let inputfile = InputFile::FileWithFormat(temp_file_path.into(), InputFormat::Json); open_editor(temp_file_path)?; let after_edit_mod_time = temp_file_path.metadata().ok().map(|m| m.modified().ok()); @@ -94,7 +98,7 @@ pub(crate) async fn cmd(cli: &UnitCtl, output_format: OutputFormat) -> Result<() }; // Send edited file to Unit to overwrite current configuration - send_and_validate_config_deserialize_response(&client, "PUT", "/config", Some(&inputfile)) + send_config_deserialize_response(&client, "PUT", "/config", &inputfile) .await .and_then(|status| output_format.write_to_stdout(&status)) } diff --git a/tools/unitctl/unitctl/src/cmd/execute.rs b/tools/unitctl/unitctl/src/cmd/execute.rs index 3a4a8b451..4e4df4d2f 100644 --- a/tools/unitctl/unitctl/src/cmd/execute.rs +++ b/tools/unitctl/unitctl/src/cmd/execute.rs @@ -1,6 +1,6 @@ use crate::inputfile::InputFile; use crate::requests::{ - send_and_validate_config_deserialize_response, send_and_validate_pem_data_deserialize_response, + send_and_validate_pem_data_deserialize_response, send_config_deserialize_response, send_body_deserialize_response, send_empty_body_deserialize_response, }; use crate::unitctl::UnitCtl; @@ -71,7 +71,7 @@ async fn send_and_deserialize( match input_file { Some(input_file) => { if input_file.is_config() { - send_and_validate_config_deserialize_response(&client, method.as_str(), path, Some(&input_file)).await + send_config_deserialize_response(&client, method.as_str(), path, &input_file).await // TLS certificate data } else if input_file.is_pem_bundle() { send_and_validate_pem_data_deserialize_response(&client, method.as_str(), path, &input_file).await diff --git a/tools/unitctl/unitctl/src/cmd/import.rs b/tools/unitctl/unitctl/src/cmd/import.rs index 956832f32..e4726a83a 100644 --- a/tools/unitctl/unitctl/src/cmd/import.rs +++ b/tools/unitctl/unitctl/src/cmd/import.rs @@ -100,13 +100,7 @@ async fn process_entry(entry: DirEntry, client: &UnitClient) -> Result<(), Unitc let result = match upload_format { UploadFormat::Config => { - requests::send_and_validate_config_deserialize_response( - client, - "PUT", - upload_path.as_str(), - Some(&input_file), - ) - .await + requests::send_config_deserialize_response(client, "PUT", upload_path.as_str(), &input_file).await } UploadFormat::PemBundle => { requests::send_and_validate_pem_data_deserialize_response(client, "PUT", upload_path.as_str(), &input_file) diff --git a/tools/unitctl/unitctl/src/inputfile.rs b/tools/unitctl/unitctl/src/inputfile.rs index 2860d2f7f..d63747e9c 100644 --- a/tools/unitctl/unitctl/src/inputfile.rs +++ b/tools/unitctl/unitctl/src/inputfile.rs @@ -144,36 +144,50 @@ impl InputFile { } } - pub fn to_unit_serializable_map(&self) -> Result { - let reader: Box = self.try_into()?; - let body_data: UnitSerializableMap = match self.format() { - InputFormat::Json => serde_json::from_reader(reader) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?, + /// The media type and the bytes to PUT for a configuration input. + /// + /// The type is the type of the body, not of the file: a JSON5 file is + /// parsed and sent as JSON, so it goes out as "application/json". + /// + /// JSON is sent as it was written. unitctl does not parse it, so an + /// operator's duplicate member reaches the server, which refuses it + /// (src/nxt_conf.c:1616), and a number keeps the spelling the file used. + /// Parsing here collapsed both silently. + /// + /// JSON5 is not JSON, so it is still parsed and re-serialized. That is + /// the one input whose bytes cannot go out unchanged. + pub fn to_config_body(&self) -> Result<(String, KnownSize), UnitctlError> { + let json = "application/json".to_string(); + + match self.format() { + InputFormat::Json => Ok((json, self.try_into()?)), InputFormat::Json5 => { - let mut reader = BufReader::new(reader); - let mut json5_string: String = String::new(); + let mut reader: Box = self.try_into()?; + let mut json5_string = String::new(); reader .read_to_string(&mut json5_string) .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?; - json5::from_str(&json5_string) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })? + let parsed: UnitSerializableMap = json5::from_str(&json5_string) + .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?; + let body = serde_json::to_string(&parsed) + .map_err(|e| UnitctlError::SerializationError { message: e.to_string() })?; + Ok((json, KnownSize::String(body))) } - // Refuse hjson and YAML by name. Handing the file to the JSON - // parser would report a syntax error on the first comment or + // Refuse hjson and YAML by name. Sending the file as it is would + // make the server report a syntax error on the first comment or // unquoted key, and that error does not tell the user what to do. InputFormat::Hjson => Err(UnitctlError::DeserializationError { message: "hjson is no longer supported: convert the file to JSON first".to_string(), - })?, + }), InputFormat::Yaml => Err(UnitctlError::DeserializationError { message: "YAML is no longer supported: convert the file to JSON first, for example with \ \"yq -o=json\"" .to_string(), - })?, + }), _ => Err(UnitctlError::DeserializationError { - message: format!("Unsupported input format for serialization: {:?}", self), - })?, - }; - Ok(body_data) + message: format!("Unsupported input format for a configuration: {:?}", self), + }), + } } } @@ -307,18 +321,42 @@ mod tests { file } + /// Read a body back without going through `into_full_body`, so the check + /// does not depend on the same code it is checking. + fn body_bytes(known_size: KnownSize) -> Vec { + match known_size { + KnownSize::Vec(bytes) => bytes, + KnownSize::String(text) => text.into_bytes(), + KnownSize::Read(mut reader, _) => { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).expect("reading the body"); + bytes + } + KnownSize::Empty => Vec::new(), + } + } + + fn config_body(file: &tempfile::NamedTempFile, format: InputFormat) -> Vec { + let (mime_type, body) = InputFile::FileWithFormat(file.path().into(), format) + .to_config_body() + .expect("the input must produce a body"); + + // Every configuration body is JSON by the time it leaves, whatever the + // file it came from. + assert_eq!(mime_type, "application/json"); + + body_bytes(body) + } + fn members_of(file: &tempfile::NamedTempFile, format: InputFormat) -> Vec { - InputFile::FileWithFormat(file.path().into(), format) - .to_unit_serializable_map() - .expect("the input must deserialize") - .keys() - .cloned() - .collect() + let body = config_body(file, format); + let map: UnitSerializableMap = serde_json::from_slice(&body).expect("the body must be JSON"); + map.keys().cloned().collect() } - /// `UnitSerializableMap` is a `serde_json::Map`, which keeps member order - /// only because every format feeding it keeps it too. `json5` reaches the - /// type by a different route than `serde_json`, so both are checked. + /// The body Unit receives keeps member order. JSON is sent as it was + /// written, so it can only keep it. JSON5 is parsed and re-serialized, and + /// both ends of that have to keep it. #[test] fn every_input_format_keeps_member_order() { let json = MEMBERS @@ -332,15 +370,51 @@ mod tests { assert_eq!(members_of(&write_input(&json, ".json5"), InputFormat::Json5), MEMBERS); } + /// A JSON file is sent byte for byte. Parsing it here kept the last of two + /// members with the same name and re-spelled every number, so the server + /// never saw what the operator wrote. The server refuses the duplicate + /// itself (src/nxt_conf.c:1616). + #[test] + fn a_json_input_is_sent_unchanged() { + let written = "{\n \"listeners\": {},\n \"listeners\": {\"*:8080\": {}},\n \"settings\": 1.50\n}\n"; + let file = write_input(written, ".json"); + + let body = config_body(&file, InputFormat::Json); + + assert_eq!(String::from_utf8(body).expect("the body is UTF-8"), written); + } + + /// Bytes that are not valid UTF-8 now reach the server, which refuses them + /// and names the member (src/nxt_conf_validation.c:1840). unitctl refused + /// them first, with a parse error that said nothing about the encoding. + #[test] + fn a_file_that_is_not_valid_utf8_is_sent_to_the_server() { + let mut file = tempfile::Builder::new() + .suffix(".json") + .tempfile() + .expect("a temporary file"); + let mut raw = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); + raw.push(0xff); + raw.extend_from_slice(br#""},"action":{"return":204}}]}"#); + std::io::Write::write_all(file.as_file_mut(), &raw).expect("writing the input"); + + let (_, body) = InputFile::FileWithFormat(file.path().into(), InputFormat::Json) + .to_config_body() + .expect("the bytes must be sent, not judged here"); + let body = body_bytes(body); + + assert_eq!(body, raw); + } + /// hjson is refused by name. Handing the file to the JSON parser instead /// would report a syntax error on the first comment, which does not tell /// the user what to do. #[test] fn an_hjson_input_is_refused_with_a_message_about_hjson() { let file = write_input("{\n # a comment\n routes: []\n}\n", ".hjson"); - let error = InputFile::from(file.path()) - .to_unit_serializable_map() - .expect_err("hjson must be refused"); + let Err(error) = InputFile::from(file.path()).to_config_body() else { + panic!("hjson must be refused"); + }; assert!( error.to_string().contains("hjson is no longer supported"), @@ -361,7 +435,9 @@ mod tests { assert!(input.is_config(), "{} must stay a config format", suffix); - let error = input.to_unit_serializable_map().expect_err("YAML must be refused"); + let Err(error) = input.to_config_body() else { + panic!("{} must be refused", suffix); + }; assert!( error.to_string().contains("YAML is no longer supported"), "unexpected message: {}", @@ -370,30 +446,11 @@ mod tests { } } - /// The way back in is strict, which is why `unitctl export` warns that the - /// verbatim backup it writes for a configuration holding bytes that are not - /// valid UTF-8 is one `unitctl import` cannot replay. - #[test] - fn a_file_that_is_not_valid_utf8_is_refused_on_the_way_in() { - let mut file = tempfile::Builder::new() - .suffix(".json") - .tempfile() - .expect("a temporary file"); - let mut raw = br#"{"routes":[{"match":{"uri":"/"#.to_vec(); - raw.push(0xff); - raw.extend_from_slice(br#""},"action":{"return":204}}]}"#); - std::io::Write::write_all(file.as_file_mut(), &raw).expect("writing the input"); - - assert!(InputFile::FileWithFormat(file.path().into(), InputFormat::Json) - .to_unit_serializable_map() - .is_err()); - } - #[test] fn an_unsupported_input_format_is_refused() { let file = write_input("not a configuration", ".txt"); assert!(InputFile::FileWithFormat(file.path().into(), InputFormat::Pem) - .to_unit_serializable_map() + .to_config_body() .is_err()); } } diff --git a/tools/unitctl/unitctl/src/known_size.rs b/tools/unitctl/unitctl/src/known_size.rs index 0de9ec814..c6e817c5b 100644 --- a/tools/unitctl/unitctl/src/known_size.rs +++ b/tools/unitctl/unitctl/src/known_size.rs @@ -126,8 +126,14 @@ mod tests { let data = b"hello, world!"; let reader = io::Cursor::new(data.to_vec()); let known = KnownSize::Read(Box::new(reader), data.len() as u64); - let (_, len) = known.into_full_body().expect("should succeed"); + let (body, len) = known.into_full_body().expect("should succeed"); assert_eq!(len, data.len() as u64); + // The length alone would pass for any thirteen bytes, and an ordinary + // JSON configuration file now travels this variant. + let collected = futures::executor::block_on(http_body_util::BodyExt::collect(body)) + .expect("collecting the body") + .to_bytes(); + assert_eq!(collected.as_ref(), data.as_slice()); } #[test] diff --git a/tools/unitctl/unitctl/src/requests.rs b/tools/unitctl/unitctl/src/requests.rs index 1317b9d0b..28a382c8a 100644 --- a/tools/unitctl/unitctl/src/requests.rs +++ b/tools/unitctl/unitctl/src/requests.rs @@ -15,31 +15,22 @@ use std::io::ErrorKind; use std::sync::atomic::AtomicUsize; use unit_client_rs::unit_client::UnitClientError; -/// Send the contents of a file to the unit server -/// We assume that the file is valid and can be sent to the server -pub async fn send_and_validate_config_deserialize_response( +/// Send a configuration file to the unit server. +/// +/// The bytes go out as they were written, for every format but JSON5. unitctl +/// used to parse the file into a `serde_json::Map` and re-serialize it. That +/// dropped an operator's duplicate member before the server could refuse it +/// (src/nxt_conf.c:1616) and re-spelled every number. The server validates: +/// UTF-8 (src/nxt_conf_validation.c:1840), duplicates, and the schema. +pub async fn send_config_deserialize_response( client: &UnitClient, method: &str, path: &str, - input_file: Option<&InputFile>, + input_file: &InputFile, ) -> Result { - let body_data = match input_file { - Some(input) => Some(input.to_unit_serializable_map()?), - None => None, - }; - - /* Unfortunately, we have load the json text into memory before sending it to the server. - * This allows for validation of the json content before sending to the server. There may be - * a better way of doing this and it is worth investigating. */ - let json = serde_json::to_value(&body_data).map_err(|error| UnitClientError::JsonError { - source: error, - path: path.into(), - })?; - - let mime_type = input_file.map(|f| f.mime_type()); - let reader = KnownSize::String(json.to_string()); + let (mime_type, body) = input_file.to_config_body()?; - streaming_upload_deserialize_response(client, method, path, mime_type, reader) + streaming_upload_deserialize_response(client, method, path, Some(mime_type), body) .await .map_err(|e| UnitctlError::UnitClientError { source: e }) } From 85e481cf94614d3042ad533ca8ad5022e609ed33 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 02:37:37 +0200 Subject: [PATCH 182/391] docs(changes): record the unitctl configuration byte path The duplicate member that used to pass now fails, and "edit" no longer takes JSON5 comments, so both belong in the release notes. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 8 ++++++++ docs/changes.xml | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGES b/CHANGES index 03258b3e4..8a2b0b8c2 100644 --- a/CHANGES +++ b/CHANGES @@ -36,6 +36,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 socket address; it refuses one with a message naming the feature. A Unix socket or a plain "http://" address is unaffected, and a build with default features behaves as before. + *) Change: unitctl sends a JSON configuration file as it was written. It + parsed the file and re-serialized it before, which kept the last of two + object members with the same name and re-spelled numbers, so the server + never saw what the operator wrote. Unit now reports the duplicate + itself, with the line and column of the file. JSON5 input is still + parsed, because it is not JSON. A file that is not valid UTF-8 is no + longer refused by unitctl; Unit refuses it and names the member. The + "edit" command no longer accepts JSON5 comments in the temporary file. *) Feature: conditional requests for static files. A "share" action now answers a GET or HEAD carrying "If-None-Match" or "If-Modified-Since" diff --git a/docs/changes.xml b/docs/changes.xml index c78116730..fed5ffdc4 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -327,6 +327,19 @@ invalidated. + + +unitctl sends a JSON configuration file as it was written. It parsed the file +and re-serialized it before, which kept the last of two object members with the +same name and re-spelled numbers, so the server never saw what the operator +wrote. Unit now reports the duplicate itself, with the line and column of the +file. JSON5 input is still parsed, because it is not JSON. A file that is not +valid UTF-8 is no longer refused by unitctl; Unit refuses it and names the +member. The "edit" command no longer accepts JSON5 comments in the temporary +file. + + + From abe1d52d8ef91b3eb93ab24b30bafa1f86f28598 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 04:24:38 +0200 Subject: [PATCH 183/391] unitctl: stop pointing at curl for a configuration it cannot send "unitctl save" warns, for a configuration holding bytes that are not valid UTF-8, that "unitctl import" cannot replay the file and tells the operator to restore it with curl instead. Curl cannot do it either. Both reach the same control API, and nxt_conf_validate_encoding() runs there before the schema is looked at, on POST and on PUT alike (nxt_controller.c, in nxt_controller_process_config()). A configuration holding those bytes is refused whichever tool sends it, so the advice named the one alternative that is guaranteed to fail. There is no route back by design. Such a value survives only because it was loaded from the state file, which keeps it exactly as written rather than repairing it -- silently rewriting an operator's value is how a working non-UTF-8 "share" path becomes a broken one -- and the controller logs it once, naming it with a pointer. The API stores no more of them. So say that, and send the reader somewhere real: the value the server named is what has to be corrected before the file can be imported. Co-Authored-By: Claude Opus 5 --- tools/unitctl/unitctl/src/cmd/save.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/unitctl/unitctl/src/cmd/save.rs b/tools/unitctl/unitctl/src/cmd/save.rs index b0efb9fb3..6ee89b4cb 100644 --- a/tools/unitctl/unitctl/src/cmd/save.rs +++ b/tools/unitctl/unitctl/src/cmd/save.rs @@ -50,7 +50,17 @@ fn prettify_or_keep(raw: &[u8], path: &str) -> Result, UnitctlError> { "Warning: {} holds bytes that are not valid UTF-8; saving what the server sent, verbatim", path ); - eprintln!("Warning: 'unitctl import' cannot replay this file -- restore it with curl"); + // Not a route around it: curl reaches the same control API, which + // refuses a configuration holding these bytes on POST and on PUT + // alike (nxt_conf_validate_encoding() in nxt_controller.c, before + // the schema is even looked at). A value like this only survives + // because it was loaded from the state file, and the server names + // it in the warning it logged then. + eprintln!( + "Warning: the control API refuses a configuration holding these bytes, so this \ + file cannot be sent back by any route; correct the value the server named in \ + its warning, then import" + ); Ok(raw.to_vec()) } } From 430323faf3b9dfb1cc29d8557312b0be8679e548 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 04:04:43 +0200 Subject: [PATCH 184/391] chore: remove the GnuTLS, CyaSSL, and PolarSSL TLS backends --gnutls, --cyassl, and --polarssl are documented-looking configure flags that cannot build. All three source files read a c->u.ssltls union member that does not exist (nxt_conn_t's union only has `tls`), and reference types such as nxt_ssltls_conf_t and nxt_event_conn_io_t that predate a refactor still visible in the current tree. Passing any of these flags reaches a compiler error instead of a clean "unsupported", confirmed against the current tree and independently reported upstream in nginx/unit#1209. CyaSSL was renamed wolfSSL and PolarSSL became mbedTLS, both in 2015, so the flags also target products that no longer exist under these names. None of the three files have been touched since the initial import; the OpenSSL backend and its abstraction in nxt_tls.h are unaffected, and the object set produced by a default build is unchanged since these files were never part of it. Co-Authored-By: Claude Sonnet 5 --- CHANGES | 13 + auto/options | 6 - auto/sources | 18 -- auto/ssltls | 136 --------- configure | 6 +- docs/changes.xml | 15 + src/nxt_cyassl.c | 621 ------------------------------------- src/nxt_gnutls.c | 742 --------------------------------------------- src/nxt_polarssl.c | 118 ------- src/nxt_service.c | 10 - src/nxt_tls.h | 13 - 11 files changed, 30 insertions(+), 1668 deletions(-) delete mode 100644 src/nxt_cyassl.c delete mode 100644 src/nxt_gnutls.c delete mode 100644 src/nxt_polarssl.c diff --git a/CHANGES b/CHANGES index 8a2b0b8c2..ebf525751 100644 --- a/CHANGES +++ b/CHANGES @@ -259,6 +259,19 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 matches. The tag keeps its format, so nothing already held in a cache is invalidated. + *) Change: removed the GnuTLS, CyaSSL, and PolarSSL backends + (nxt_gnutls.c, nxt_cyassl.c, nxt_polarssl.c) and their configure + plumbing ("--gnutls", "--cyassl", "--polarssl"). None of the + three ever built: they read a "c->u.ssltls" union member that + does not exist, and reference types such as "nxt_ssltls_conf_t" + and "nxt_event_conn_io_t" that predate a refactor still visible + in the current tree, so passing the flag reaches a compiler + error instead of a clean "unsupported". CyaSSL was renamed + wolfSSL and PolarSSL became mbedTLS, both in 2015, so the flags + also target products that no longer exist under these names. + The OpenSSL backend is unaffected; none of these files were + ever part of a default build. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/auto/options b/auto/options index f19948588..bb7beda17 100644 --- a/auto/options +++ b/auto/options @@ -22,9 +22,6 @@ NXT_TRY_PCRE2=YES NXT_TLS=NO NXT_OPENSSL=NO -NXT_GNUTLS=NO -NXT_CYASSL=NO -NXT_POLARSSL=NO NXT_ZLIB=NO NXT_ZSTD=NO @@ -112,9 +109,6 @@ do --no-pcre2) NXT_TRY_PCRE2=NO ;; --openssl) NXT_OPENSSL=YES ;; - --gnutls) NXT_GNUTLS=YES ;; - --cyassl) NXT_CYASSL=YES ;; - --polarssl) NXT_POLARSSL=YES ;; --zlib) NXT_ZLIB=YES ;; --zstd) NXT_ZSTD=YES ;; diff --git a/auto/sources b/auto/sources index ad9fbfce2..ec2ab33fe 100644 --- a/auto/sources +++ b/auto/sources @@ -112,9 +112,6 @@ NXT_LIB_UNIT_SRCS="src/nxt_unit.c" NXT_LIB_TLS_DEPS="src/nxt_tls.h" NXT_LIB_TLS_SRCS="src/nxt_cert.c" NXT_LIB_OPENSSL_SRCS="src/nxt_openssl.c" -NXT_LIB_GNUTLS_SRCS="src/nxt_gnutls.c" -NXT_LIB_CYASSL_SRCS="src/nxt_cyassl.c" -NXT_LIB_POLARSSL_SRCS="src/nxt_polarssl.c" NXT_LIB_PCRE_SRCS="src/nxt_pcre.c" NXT_LIB_PCRE2_SRCS="src/nxt_pcre2.c" @@ -216,21 +213,6 @@ if [ $NXT_OPENSSL = YES ]; then fi -if [ $NXT_GNUTLS = YES ]; then - NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_GNUTLS_SRCS" -fi - - -if [ $NXT_CYASSL = YES ]; then - NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_CYASSL_SRCS" -fi - - -if [ $NXT_POLARSSL = YES ]; then - NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_POLARSSL_SRCS" -fi - - if [ "$NXT_HAVE_ZLIB" = "YES" ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_zlib.c" fi diff --git a/auto/ssltls b/auto/ssltls index c4447e0c2..ac829176f 100644 --- a/auto/ssltls +++ b/auto/ssltls @@ -5,13 +5,6 @@ NXT_OPENSSL_CFLAGS= NXT_OPENSSL_LIBS= -NXT_GNUTLS_CFLAGS= -NXT_GNUTLS_LIBS= -NXT_OPENSSL_LIBS= -NXT_CYASSL_CFLAGS= -NXT_CYASSL_LIBS= -NXT_POLARSSL_CFLAGS= -NXT_POLARSSL_LIBS= if [ $NXT_OPENSSL = YES ]; then @@ -124,132 +117,3 @@ if [ $NXT_OPENSSL = YES ]; then }" . auto/feature fi - - -if [ $NXT_GNUTLS = YES ]; then - - if /bin/sh -c "(pkg-config gnutls --exists)" >> $NXT_AUTOCONF_ERR 2>&1; - then - NXT_GNUTLS_CFLAGS=`pkg-config gnutls --cflags` - NXT_GNUTLS_LIBS=`pkg-config gnutls --libs` - - nxt_feature="GnuTLS library" - nxt_feature_name=NXT_HAVE_GNUTLS - nxt_feature_run=yes - nxt_feature_incs=$NXT_GNUTLS_CFLAGS - nxt_feature_libs=$NXT_GNUTLS_LIBS - nxt_feature_test="#include - - int main(void) { - gnutls_global_init(); - gnutls_global_deinit(); - return 0; - }" - . auto/feature - - - if [ $nxt_found = yes ]; then - NXT_TLS=YES - - $echo " + GnuTLS version: `pkg-config gnutls --modversion`" - - - nxt_feature="gnutls_transport_set_vec_push_function" - nxt_feature_name=NXT_HAVE_GNUTLS_VEC_PUSH - nxt_feature_run=no - nxt_feature_incs=$NXT_GNUTLS_CFLAGS - nxt_feature_libs=$NXT_GNUTLS_LIBS - nxt_feature_test="#include - - int main(void) { - gnutls_transport_set_vec_push_function(NULL, NULL); - return 0; - }" - . auto/feature - - - nxt_feature="gnutls_global_set_time_function" - nxt_feature_name=NXT_HAVE_GNUTLS_SET_TIME - nxt_feature_run=no - nxt_feature_incs=$NXT_GNUTLS_CFLAGS - nxt_feature_libs=$NXT_GNUTLS_LIBS - nxt_feature_test="#include - - int main(void) { - gnutls_global_set_time_function(NULL); - return 0; - }" - . auto/feature - - else - $echo - $echo $0: error: no GnuTLS library found. - $echo - exit 1; - fi - fi -fi - - -if [ $NXT_CYASSL = YES ]; then - - nxt_feature="CyaSSL library" - nxt_feature_name=NXT_HAVE_CYASSL - nxt_feature_run=yes - nxt_feature_incs= - nxt_feature_libs="-lcyassl" - nxt_feature_test="#include - - int main(void) { - CyaSSL_Init(); - CyaSSL_Cleanup(); - return 0; - }" - . auto/feature - - - if [ $nxt_found = yes ]; then - NXT_TLS=YES - NXT_CYASSL_CFLAGS="$nxt_feature_incs" - NXT_CYASSL_LIBS="$nxt_feature_libs" - - else - $echo - $echo $0: error: no CyaSSL library found. - $echo - exit 1; - fi -fi - - -if [ $NXT_POLARSSL = YES ]; then - - nxt_feature="PolarSSL library" - nxt_feature_name=NXT_HAVE_POLARSSL - nxt_feature_run=yes - nxt_feature_incs= - nxt_feature_libs="-lpolarssl" - nxt_feature_test="#include - - int main(void) { - ssl_context ssl; - memset(&ssl, '\0', sizeof(ssl)); - ssl_init(&ssl); - ssl_free(&ssl); - return 0; - }" - . auto/feature - - - if [ $nxt_found = yes ]; then - NXT_TLS=YES - NXT_POLARSSL_CFLAGS="$nxt_feature_incs" - NXT_POLARSSL_LIBS="$nxt_feature_libs" - - else - $echo - $echo $0: error: no PolarSSL library found. - $echo - exit 1; - fi -fi diff --git a/configure b/configure index aba56e8ae..44eb5d8f6 100755 --- a/configure +++ b/configure @@ -168,13 +168,11 @@ END # LOOK -NXT_LIB_AUX_CFLAGS="$NXT_OPENSSL_CFLAGS $NXT_GNUTLS_CFLAGS \\ - $NXT_CYASSL_CFLAGS $NXT_POLARSSL_CFLAGS \\ +NXT_LIB_AUX_CFLAGS="$NXT_OPENSSL_CFLAGS \\ $NXT_PCRE_CFLAGS $NXT_ZLIB_CFLAGS $NXT_ZSTD_CFLAGS \\ $NXT_BROTLI_CFLAGS" -NXT_LIB_AUX_LIBS="$NXT_OPENSSL_LIBS $NXT_GNUTLS_LIBS \\ - $NXT_CYASSL_LIBS $NXT_POLARSSL_LIBS \\ +NXT_LIB_AUX_LIBS="$NXT_OPENSSL_LIBS \\ $NXT_PCRE_LIB $NXT_ZLIB_LIBS $NXT_ZSTD_LIBS \\ $NXT_BROTLI_LIBS" diff --git a/docs/changes.xml b/docs/changes.xml index fed5ffdc4..d4e1f8bb2 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -340,6 +340,21 @@ file. + + +removed the GnuTLS, CyaSSL, and PolarSSL backends (nxt_gnutls.c, +nxt_cyassl.c, nxt_polarssl.c) and their configure plumbing ("--gnutls", +"--cyassl", "--polarssl"). None of the three ever built: they read a +"c->u.ssltls" union member that does not exist, and reference types such +as "nxt_ssltls_conf_t" and "nxt_event_conn_io_t" that predate a refactor +still visible in the current tree, so passing the flag reaches a compiler +error instead of a clean "unsupported". CyaSSL was renamed wolfSSL and +PolarSSL became mbedTLS, both in 2015, so the flags also target products +that no longer exist under these names. The OpenSSL backend is +unaffected; none of these files were ever part of a default build. + + + diff --git a/src/nxt_cyassl.c b/src/nxt_cyassl.c deleted file mode 100644 index 91039e0c5..000000000 --- a/src/nxt_cyassl.c +++ /dev/null @@ -1,621 +0,0 @@ - -/* - * Copyright (C) NGINX, Inc. - * Copyright (C) Igor Sysoev - */ - -#include -#include -#include - - -typedef struct { - CYASSL *session; - - int ssl_error; - uint8_t times; /* 2 bits */ - - nxt_buf_mem_t buffer; -} nxt_cyassl_conn_t; - - -static nxt_int_t nxt_cyassl_server_init(nxt_ssltls_conf_t *conf); -static void nxt_cyassl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c); -static void nxt_cyassl_session_cleanup(void *data); -static int nxt_cyassl_io_recv(CYASSL *ssl, char *buf, int size, void *data); -static int nxt_cyassl_io_send(CYASSL *ssl, char *buf, int size, void *data); -static void nxt_cyassl_conn_handshake(nxt_thread_t *thr, void *obj, void *data); -static void nxt_cyassl_conn_io_read(nxt_thread_t *thr, void *obj, void *data); -static void nxt_cyassl_conn_io_shutdown(nxt_thread_t *thr, void *obj, - void *data); -static ssize_t nxt_cyassl_conn_io_write_chunk(nxt_thread_t *thr, - nxt_event_conn_t *c, nxt_buf_t *b, size_t limit); -static ssize_t nxt_cyassl_conn_io_send(nxt_event_conn_t *c, void *buf, - size_t size); -static nxt_int_t nxt_cyassl_conn_test_error(nxt_thread_t *thr, - nxt_event_conn_t *c, int err, nxt_work_handler_t handler); -static void nxt_cdecl nxt_cyassl_conn_error(nxt_event_conn_t *c, nxt_err_t err, - const char *fmt, ...); -static nxt_uint_t nxt_cyassl_log_error_level(nxt_event_conn_t *c, nxt_err_t err, - int ssl_error); -static void nxt_cdecl nxt_cyassl_log_error(nxt_uint_t level, nxt_log_t *log, - int ret, const char *fmt, ...); -static u_char *nxt_cyassl_copy_error(int err, u_char *p, u_char *end); - - -const nxt_ssltls_lib_t nxt_cyassl_lib = { - nxt_cyassl_server_init, - NULL, -}; - - -static nxt_event_conn_io_t nxt_cyassl_event_conn_io = { - NULL, - NULL, - - nxt_cyassl_conn_io_read, - NULL, - NULL, - - nxt_event_conn_io_write, - nxt_cyassl_conn_io_write_chunk, - NULL, - NULL, - nxt_cyassl_conn_io_send, - - nxt_cyassl_conn_io_shutdown, -}; - - -static nxt_int_t -nxt_cyassl_start(void) -{ - int err; - nxt_thread_t *thr; - static nxt_bool_t started; - - if (nxt_fast_path(started)) { - return NXT_OK; - } - - started = 1; - - thr = nxt_thread(); - - /* TODO: CyaSSL_Cleanup() */ - - err = CyaSSL_Init(); - if (err != SSL_SUCCESS) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, err, - "CyaSSL_Init() failed"); - return NXT_ERROR; - } - - nxt_thread_log_error(NXT_LOG_INFO, "CyaSSL version: %s", - LIBCYASSL_VERSION_STRING); - - /* CyaSSL_SetLoggingCb */ - /* CyaSSL_SetAllocators */ - - return NXT_OK; -} - - -static nxt_int_t -nxt_cyassl_server_init(nxt_ssltls_conf_t *conf) -{ - int err; - char *certificate, *key; - CYASSL_CTX *ctx; - nxt_thread_t *thr; - - thr = nxt_thread(); - - if (nxt_slow_path(nxt_cyassl_start() != NXT_OK)) { - return NXT_ERROR; - } - - ctx = CyaSSL_CTX_new(CyaSSLv23_server_method()); - if (ctx == NULL) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, 0, - "CyaSSL_CTX_new() failed"); - return NXT_ERROR; - } - - conf->ctx = ctx; - conf->conn_init = nxt_cyassl_conn_init; - - certificate = conf->certificate; - - err = CyaSSL_CTX_use_certificate_file(ctx, certificate, SSL_FILETYPE_PEM); - if (err != SSL_SUCCESS) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, err, - "CyaSSL_CTX_use_certificate_file(\"%s\") failed", - certificate); - goto fail; - } - - key = conf->certificate_key; - - err = CyaSSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM); - if (err != SSL_SUCCESS) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, err, - "CyaSSL_CTX_use_PrivateKey_file(\"%s\") failed", - key); - goto fail; - } - - if (conf->ciphers != NULL) { - err = CyaSSL_CTX_set_cipher_list(ctx, conf->ciphers); - if (err != SSL_SUCCESS) { - nxt_cyassl_log_error(NXT_LOG_ALERT, thr->log, err, - "CyaSSL_CTX_set_cipher_list(\"%s\") failed", - conf->ciphers); - goto fail; - } - } - - /* TODO: ca_certificate */ - - CyaSSL_SetIORecv(ctx, nxt_cyassl_io_recv); - CyaSSL_SetIOSend(ctx, nxt_cyassl_io_send); - - return NXT_OK; - -fail: - - CyaSSL_CTX_free(ctx); - - return NXT_ERROR; -} - - -static void -nxt_cyassl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c) -{ - CYASSL *s; - CYASSL_CTX *ctx; - nxt_cyassl_conn_t *ssltls; - nxt_mem_pool_cleanup_t *mpcl; - - nxt_log_debug(c->socket.log, "cyassl conn init"); - - ssltls = nxt_mp_zget(c->mem_pool, sizeof(nxt_cyassl_conn_t)); - if (ssltls == NULL) { - goto fail; - } - - c->u.ssltls = ssltls; - nxt_buf_mem_set_size(&ssltls->buffer, conf->buffer_size); - - mpcl = nxt_mem_pool_cleanup(c->mem_pool, 0); - if (mpcl == NULL) { - goto fail; - } - - ctx = conf->ctx; - - s = CyaSSL_new(ctx); - if (s == NULL) { - nxt_cyassl_log_error(NXT_LOG_ALERT, c->socket.log, 0, - "CyaSSL_new() failed"); - goto fail; - } - - ssltls->session = s; - mpcl->handler = nxt_cyassl_session_cleanup; - mpcl->data = ssltls; - - CyaSSL_SetIOReadCtx(s, c); - CyaSSL_SetIOWriteCtx(s, c); - - c->io = &nxt_cyassl_event_conn_io; - c->sendfile = NXT_CONN_SENDFILE_OFF; - - nxt_cyassl_conn_handshake(thr, c, c->socket.data); - return; - -fail: - - nxt_event_conn_io_handle(thr, c->read_work_queue, - c->read_state->error_handler, c, c->socket.data); -} - - -static void -nxt_cyassl_session_cleanup(void *data) -{ - nxt_cyassl_conn_t *ssltls; - - ssltls = data; - - nxt_thread_log_debug("cyassl session cleanup"); - - nxt_free(ssltls->buffer.start); - - CyaSSL_free(ssltls->session); -} - - -static int -nxt_cyassl_io_recv(CYASSL *ssl, char *buf, int size, void *data) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - n = thr->engine->event->io->recv(c, (u_char *) buf, size, 0); - - if (n > 0) { - return n; - } - - if (n == 0) { - return CYASSL_CBIO_ERR_CONN_CLOSE; - } - - if (n == NXT_AGAIN) { - return CYASSL_CBIO_ERR_WANT_READ; - } - - return CYASSL_CBIO_ERR_GENERAL; -} - - -static int -nxt_cyassl_io_send(CYASSL *ssl, char *buf, int size, void *data) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - n = thr->engine->event->io->send(c, (u_char *) buf, size); - - if (n > 0) { - return n; - } - - if (n == NXT_AGAIN) { - return CYASSL_CBIO_ERR_WANT_WRITE; - } - - return CYASSL_CBIO_ERR_GENERAL; -} - - -static void -nxt_cyassl_conn_handshake(nxt_thread_t *thr, void *obj, void *data) -{ - int ret; - nxt_int_t n; - nxt_err_t err; - nxt_event_conn_t *c; - nxt_cyassl_conn_t *ssltls; - - c = obj; - ssltls = c->u.ssltls; - - nxt_log_debug(thr->log, "cyassl conn handshake: %d", ssltls->times); - - /* "ssltls->times == 1" is suitable to run CyaSSL_negotiate() in job. */ - - ret = CyaSSL_negotiate(ssltls->session); - - err = (ret != 0) ? nxt_socket_errno : 0; - - nxt_thread_time_debug_update(thr); - - nxt_log_debug(thr->log, "CyaSSL_negotiate(%d): %d", c->socket.fd, ret); - - if (ret == 0) { - nxt_cyassl_conn_io_read(thr, c, data); - return; - } - - n = nxt_cyassl_conn_test_error(thr, c, ret, nxt_cyassl_conn_handshake); - - if (n == NXT_ERROR) { - nxt_cyassl_conn_error(c, err, "CyaSSL_negotiate(%d) failed", - c->socket.fd); - - nxt_event_conn_io_handle(thr, c->read_work_queue, - c->read_state->error_handler, c, data); - - } else if (ssltls->ssl_error == SSL_ERROR_WANT_READ && ssltls->times < 2) { - ssltls->times++; - } -} - - -static void -nxt_cyassl_conn_io_read(nxt_thread_t *thr, void *obj, void *data) -{ - int ret; - nxt_buf_t *b; - nxt_err_t err; - nxt_int_t n; - nxt_event_conn_t *c; - nxt_cyassl_conn_t *ssltls; - nxt_work_handler_t handler; - - c = obj; - - nxt_log_debug(thr->log, "cyassl conn read"); - - handler = c->read_state->ready_handler; - b = c->read; - - /* b == NULL is used to test descriptor readiness. */ - - if (b != NULL) { - ssltls = c->u.ssltls; - - ret = CyaSSL_read(ssltls->session, b->mem.free, - b->mem.end - b->mem.free); - - err = (ret <= 0) ? nxt_socket_errno : 0; - - nxt_log_debug(thr->log, "CyaSSL_read(%d, %p, %uz): %d", - c->socket.fd, b->mem.free, b->mem.end - b->mem.free, ret); - - if (ret > 0) { - /* c->socket.read_ready is kept. */ - b->mem.free += ret; - handler = c->read_state->ready_handler; - - } else { - n = nxt_cyassl_conn_test_error(thr, c, ret, - nxt_cyassl_conn_io_read); - - if (nxt_fast_path(n != NXT_ERROR)) { - return; - } - - nxt_cyassl_conn_error(c, err, "CyaSSL_read(%d, %p, %uz) failed", - c->socket.fd, b->mem.free, - b->mem.end - b->mem.free); - - handler = c->read_state->error_handler; - } - } - - nxt_event_conn_io_handle(thr, c->read_work_queue, handler, c, data); -} - - -static ssize_t -nxt_cyassl_conn_io_write_chunk(nxt_thread_t *thr, nxt_event_conn_t *c, - nxt_buf_t *b, size_t limit) -{ - nxt_cyassl_conn_t *ssltls; - - nxt_log_debug(thr->log, "cyassl conn write chunk"); - - ssltls = c->u.ssltls; - - return nxt_sendbuf_copy_coalesce(c, &ssltls->buffer, b, limit); -} - - -static ssize_t -nxt_cyassl_conn_io_send(nxt_event_conn_t *c, void *buf, size_t size) -{ - int ret; - nxt_err_t err; - nxt_int_t n; - nxt_cyassl_conn_t *ssltls; - - nxt_log_debug(c->socket.log, "cyassl send"); - - ssltls = c->u.ssltls; - - ret = CyaSSL_write(ssltls->session, buf, size); - - if (ret <= 0) { - err = nxt_socket_errno; - c->socket.error = err; - - } else { - err = 0; - } - - nxt_log_debug(c->socket.log, "CyaSSL_write(%d, %p, %uz): %d", - c->socket.fd, buf, size, ret); - - if (ret > 0) { - return ret; - } - - n = nxt_cyassl_conn_test_error(nxt_thread(), c, ret, - nxt_event_conn_io_write); - - if (nxt_slow_path(n == NXT_ERROR)) { - nxt_cyassl_conn_error(c, err, "CyaSSL_write(%d, %p, %uz) failed", - c->socket.fd, buf, size); - } - - return n; -} - - -static void -nxt_cyassl_conn_io_shutdown(nxt_thread_t *thr, void *obj, void *data) -{ - int ret; - nxt_event_conn_t *c; - nxt_cyassl_conn_t *ssltls; - - c = obj; - - nxt_log_debug(thr->log, "cyassl conn shutdown"); - - ssltls = c->u.ssltls; - - ret = CyaSSL_shutdown(ssltls->session); - - nxt_log_debug(thr->log, "CyaSSL_shutdown(%d): %d", c->socket.fd, ret); - - if (nxt_slow_path(ret != SSL_SUCCESS)) { - nxt_cyassl_conn_error(c, 0, "CyaSSL_shutdown(%d) failed", c->socket.fd); - } - - nxt_event_conn_io_handle(thr, c->write_work_queue, - c->write_state->close_handler, c, data); -} - - -static nxt_int_t -nxt_cyassl_conn_test_error(nxt_thread_t *thr, nxt_event_conn_t *c, int ret, - nxt_work_handler_t handler) -{ - nxt_work_queue_t *wq; - nxt_cyassl_conn_t *ssltls; - - ssltls = c->u.ssltls; - ssltls->ssl_error = CyaSSL_get_error(ssltls->session, ret); - - nxt_log_debug(thr->log, "CyaSSL_get_error(): %d", ssltls->ssl_error); - - switch (ssltls->ssl_error) { - - case SSL_ERROR_WANT_READ: - nxt_event_fd_block_write(thr->engine, &c->socket); - - c->socket.read_ready = 0; - c->socket.read_handler = handler; - - if (nxt_event_fd_is_disabled(c->socket.read)) { - nxt_event_fd_enable_read(thr->engine, &c->socket); - } - - return NXT_AGAIN; - - case SSL_ERROR_WANT_WRITE: - nxt_event_fd_block_read(thr->engine, &c->socket); - - c->socket.write_ready = 0; - c->socket.write_handler = handler; - - if (nxt_event_fd_is_disabled(c->socket.write)) { - nxt_event_fd_enable_write(thr->engine, &c->socket); - } - - return NXT_AGAIN; - - case SSL_ERROR_ZERO_RETURN: - /* A "close notify" alert */ - - if (c->read_state != NULL) { - wq = c->read_work_queue; - handler = c->read_state->close_handler; - - } else { - wq = c->write_work_queue; - handler = c->write_state->close_handler; - } - - nxt_event_conn_io_handle(thr, wq, handler, c, c->socket.data); - - return 0; - - default: - return NXT_ERROR; - } -} - - -static void nxt_cdecl -nxt_cyassl_conn_error(nxt_event_conn_t *c, nxt_err_t err, const char *fmt, ...) -{ - u_char *p, *end; - va_list args; - nxt_uint_t level; - nxt_cyassl_conn_t *ssltls; - u_char msg[NXT_MAX_ERROR_STR]; - - ssltls = c->u.ssltls; - - level = nxt_cyassl_log_error_level(c, err, ssltls->ssl_error); - - if (nxt_log_level_enough(c->socket.log, level)) { - - end = msg + sizeof(msg); - - va_start(args, fmt); - p = nxt_vsprintf(msg, end, fmt, args); - va_end(args); - - if (err != 0) { - p = nxt_sprintf(p, end, " %E", err); - } - - p = nxt_cyassl_copy_error(ssltls->ssl_error, p, end); - - nxt_log_error(level, c->socket.log, "%*s", p - msg, msg); - } -} - - -static nxt_uint_t -nxt_cyassl_log_error_level(nxt_event_conn_t *c, nxt_err_t err, int ssl_error) -{ - switch (ssl_error) { - - case SOCKET_ERROR_E: /* -208 */ - case MATCH_SUITE_ERROR: /* -261 */ - break; - - default: - return NXT_LOG_ALERT; - } - - return NXT_LOG_INFO; -} - - -static void nxt_cdecl -nxt_cyassl_log_error(nxt_uint_t level, nxt_log_t *log, int err, - const char *fmt, ...) -{ - u_char *p, *end; - va_list args; - u_char msg[NXT_MAX_ERROR_STR]; - - if (nxt_log_level_enough(log, level)) { - - end = msg + sizeof(msg); - - va_start(args, fmt); - p = nxt_vsprintf(msg, end, fmt, args); - va_end(args); - - p = nxt_cyassl_copy_error(err, p, end); - - nxt_log_error(level, log, "%*s", p - msg, msg); - } -} - - -static u_char * -nxt_cyassl_copy_error(int err, u_char *p, u_char *end) -{ - p = nxt_sprintf(p, end, " (SSL:%d ", err); - - CyaSSL_ERR_error_string_n(err, (char *) p, end - p); - - p += nxt_strlen(p); - - if (p < end) { - *p++ = ')'; - } - - return p; -} diff --git a/src/nxt_gnutls.c b/src/nxt_gnutls.c deleted file mode 100644 index aab4699c6..000000000 --- a/src/nxt_gnutls.c +++ /dev/null @@ -1,742 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) NGINX, Inc. - */ - -#include -#include - - -typedef struct { - gnutls_session_t session; - - uint8_t times; /* 2 bits */ - uint8_t no_shutdown; /* 1 bit */ - - nxt_buf_mem_t buffer; -} nxt_gnutls_conn_t; - - -typedef struct { - gnutls_priority_t ciphers; - gnutls_certificate_credentials_t certificate; -} nxt_gnutls_ctx_t; - - - -#if (NXT_HAVE_GNUTLS_SET_TIME) -time_t nxt_gnutls_time(time_t *tp); -#endif -static nxt_int_t nxt_gnutls_server_init(nxt_ssltls_conf_t *conf); -static nxt_int_t nxt_gnutls_set_ciphers(nxt_ssltls_conf_t *conf); - -static void nxt_gnutls_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c); -static void nxt_gnutls_session_cleanup(void *data); -static ssize_t nxt_gnutls_pull(gnutls_transport_ptr_t data, void *buf, - size_t size); -static ssize_t nxt_gnutls_push(gnutls_transport_ptr_t data, const void *buf, - size_t size); -#if (NXT_HAVE_GNUTLS_VEC_PUSH) -static ssize_t nxt_gnutls_vec_push(gnutls_transport_ptr_t data, - const giovec_t *iov, int iovcnt); -#endif -static void nxt_gnutls_conn_handshake(nxt_thread_t *thr, void *obj, void *data); -static void nxt_gnutls_conn_io_read(nxt_thread_t *thr, void *obj, void *data); -static ssize_t nxt_gnutls_conn_io_write_chunk(nxt_thread_t *thr, - nxt_event_conn_t *c, nxt_buf_t *b, size_t limit); -static ssize_t nxt_gnutls_conn_io_send(nxt_event_conn_t *c, void *buf, - size_t size); -static void nxt_gnutls_conn_io_shutdown(nxt_thread_t *thr, void *obj, - void *data); -static nxt_int_t nxt_gnutls_conn_test_error(nxt_thread_t *thr, - nxt_event_conn_t *c, ssize_t err, nxt_work_handler_t handler); -static void nxt_cdecl nxt_gnutls_conn_log_error(nxt_event_conn_t *c, - ssize_t err, const char *fmt, ...); -static nxt_uint_t nxt_gnutls_log_error_level(nxt_event_conn_t *c, ssize_t err); -static void nxt_cdecl nxt_gnutls_log_error(nxt_uint_t level, nxt_log_t *log, - int err, const char *fmt, ...); - - -const nxt_ssltls_lib_t nxt_gnutls_lib = { - nxt_gnutls_server_init, - NULL, -}; - - -static nxt_event_conn_io_t nxt_gnutls_event_conn_io = { - NULL, - NULL, - - nxt_gnutls_conn_io_read, - NULL, - NULL, - - nxt_event_conn_io_write, - nxt_gnutls_conn_io_write_chunk, - NULL, - NULL, - nxt_gnutls_conn_io_send, - - nxt_gnutls_conn_io_shutdown, -}; - - -static nxt_int_t -nxt_gnutls_start(void) -{ - int ret; - static nxt_bool_t started; - - if (nxt_fast_path(started)) { - return NXT_OK; - } - - started = 1; - - /* TODO: gnutls_global_deinit */ - - ret = gnutls_global_init(); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, nxt_thread_log(), ret, - "gnutls_global_init() failed"); - return NXT_ERROR; - } - - nxt_thread_log_error(NXT_LOG_INFO, "GnuTLS version: %s", - gnutls_check_version(NULL)); - -#if (NXT_HAVE_GNUTLS_SET_TIME) - gnutls_global_set_time_function(nxt_gnutls_time); -#endif - - return NXT_OK; -} - - -#if (NXT_HAVE_GNUTLS_SET_TIME) - -/* GnuTLS 2.12.0 */ - -time_t -nxt_gnutls_time(time_t *tp) -{ - time_t t; - nxt_thread_t *thr; - - thr = nxt_thread(); - nxt_log_debug(thr->log, "gnutls time"); - - t = (time_t) nxt_thread_time(thr); - - if (tp != NULL) { - *tp = t; - } - - return t; -} - -#endif - - -static nxt_int_t -nxt_gnutls_server_init(nxt_ssltls_conf_t *conf) -{ - int ret; - char *certificate, *key, *ca_certificate; - nxt_thread_t *thr; - nxt_gnutls_ctx_t *ctx; - - if (nxt_slow_path(nxt_gnutls_start() != NXT_OK)) { - return NXT_ERROR; - } - - /* TODO: mem_pool, cleanup: gnutls_certificate_free_credentials, - gnutls_priority_deinit */ - - ctx = nxt_zalloc(sizeof(nxt_gnutls_ctx_t)); - if (ctx == NULL) { - return NXT_ERROR; - } - - conf->ctx = ctx; - conf->conn_init = nxt_gnutls_conn_init; - - thr = nxt_thread(); - - ret = gnutls_certificate_allocate_credentials(&ctx->certificate); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, thr->log, ret, - "gnutls_certificate_allocate_credentials() failed"); - return NXT_ERROR; - } - - certificate = conf->certificate; - key = conf->certificate_key; - - ret = gnutls_certificate_set_x509_key_file(ctx->certificate, certificate, - key, GNUTLS_X509_FMT_PEM); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, thr->log, ret, - "gnutls_certificate_set_x509_key_file(\"%s\", \"%s\") failed", - certificate, key); - goto certificate_fail; - } - - if (nxt_gnutls_set_ciphers(conf) != NXT_OK) { - goto ciphers_fail; - } - - if (conf->ca_certificate != NULL) { - ca_certificate = conf->ca_certificate; - - ret = gnutls_certificate_set_x509_trust_file(ctx->certificate, - ca_certificate, - GNUTLS_X509_FMT_PEM); - if (ret < 0) { - nxt_gnutls_log_error(NXT_LOG_ALERT, thr->log, ret, - "gnutls_certificate_set_x509_trust_file(\"%s\") failed", - ca_certificate); - goto ca_certificate_fail; - } - } - - return NXT_OK; - -ca_certificate_fail: - - gnutls_priority_deinit(ctx->ciphers); - -ciphers_fail: - -certificate_fail: - - gnutls_certificate_free_credentials(ctx->certificate); - - return NXT_ERROR; -} - - -static nxt_int_t -nxt_gnutls_set_ciphers(nxt_ssltls_conf_t *conf) -{ - int ret; - const char *ciphers; - const char *err; - nxt_gnutls_ctx_t *ctx; - - ciphers = (conf->ciphers != NULL) ? conf->ciphers : "NORMAL:!COMP-DEFLATE"; - ctx = conf->ctx; - - ret = gnutls_priority_init(&ctx->ciphers, ciphers, &err); - - switch (ret) { - - case GNUTLS_E_SUCCESS: - return NXT_OK; - - case GNUTLS_E_INVALID_REQUEST: - nxt_gnutls_log_error(NXT_LOG_ALERT, nxt_thread_log(), ret, - "gnutls_priority_init(\"%s\") failed at \"%s\"", - ciphers, err); - return NXT_ERROR; - - default: - nxt_gnutls_log_error(NXT_LOG_ALERT, nxt_thread_log(), ret, - "gnutls_priority_init() failed"); - return NXT_ERROR; - } -} - - -static void -nxt_gnutls_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c) -{ - int ret; - gnutls_session_t sess; - nxt_gnutls_ctx_t *ctx; - nxt_gnutls_conn_t *ssltls; - nxt_mem_pool_cleanup_t *mpcl; - - nxt_log_debug(c->socket.log, "gnutls conn init"); - - ssltls = nxt_mp_zget(c->mem_pool, sizeof(nxt_gnutls_conn_t)); - if (ssltls == NULL) { - goto fail; - } - - c->u.ssltls = ssltls; - nxt_buf_mem_set_size(&ssltls->buffer, conf->buffer_size); - - mpcl = nxt_mem_pool_cleanup(c->mem_pool, 0); - if (mpcl == NULL) { - goto fail; - } - - ret = gnutls_init(&ssltls->session, GNUTLS_SERVER); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, c->socket.log, ret, - "gnutls_init() failed"); - goto fail; - } - - sess = ssltls->session; - mpcl->handler = nxt_gnutls_session_cleanup; - mpcl->data = ssltls; - - ctx = conf->ctx; - - ret = gnutls_priority_set(sess, ctx->ciphers); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, c->socket.log, ret, - "gnutls_priority_set() failed"); - goto fail; - } - - /* - * Disable TLS random padding of records in CBC ciphers, - * which may be up to 255 bytes. - */ - gnutls_record_disable_padding(sess); - - ret = gnutls_credentials_set(sess, GNUTLS_CRD_CERTIFICATE, - ctx->certificate); - if (ret != GNUTLS_E_SUCCESS) { - nxt_gnutls_log_error(NXT_LOG_ALERT, c->socket.log, ret, - "gnutls_credentials_set() failed"); - goto fail; - } - - if (conf->ca_certificate != NULL) { - gnutls_certificate_server_set_request(sess, GNUTLS_CERT_REQUEST); - } - - gnutls_transport_set_ptr(sess, (gnutls_transport_ptr_t) c); - gnutls_transport_set_pull_function(sess, nxt_gnutls_pull); - gnutls_transport_set_push_function(sess, nxt_gnutls_push); -#if (NXT_HAVE_GNUTLS_VEC_PUSH) - gnutls_transport_set_vec_push_function(sess, nxt_gnutls_vec_push); -#endif - - c->io = &nxt_gnutls_event_conn_io; - c->sendfile = NXT_CONN_SENDFILE_OFF; - - nxt_gnutls_conn_handshake(thr, c, c->socket.data); - return; - -fail: - - nxt_event_conn_io_handle(thr, c->read_work_queue, - c->read_state->error_handler, c, c->socket.data); -} - - -static void -nxt_gnutls_session_cleanup(void *data) -{ - nxt_gnutls_conn_t *ssltls; - - ssltls = data; - - nxt_thread_log_debug("gnutls session cleanup"); - - nxt_free(ssltls->buffer.start); - - gnutls_deinit(ssltls->session); -} - - -static ssize_t -nxt_gnutls_pull(gnutls_transport_ptr_t data, void *buf, size_t size) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - n = thr->engine->event->io->recv(c, buf, size, 0); - - if (n == NXT_AGAIN) { - nxt_set_errno(NXT_EAGAIN); - return -1; - } - - return n; -} - - -static ssize_t -nxt_gnutls_push(gnutls_transport_ptr_t data, const void *buf, size_t size) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - n = thr->engine->event->io->send(c, (u_char *) buf, size); - - if (n == NXT_AGAIN) { - nxt_set_errno(NXT_EAGAIN); - return -1; - } - - return n; -} - - -#if (NXT_HAVE_GNUTLS_VEC_PUSH) - -/* GnuTLS 2.12.0 */ - -static ssize_t -nxt_gnutls_vec_push(gnutls_transport_ptr_t data, const giovec_t *iov, - int iovcnt) -{ - ssize_t n; - nxt_thread_t *thr; - nxt_event_conn_t *c; - - c = data; - thr = nxt_thread(); - - /* - * This code assumes that giovec_t is the same as "struct iovec" - * and nxt_iobuf_t. It is not true for Windows. - */ - n = thr->engine->event->io->writev(c, (nxt_iobuf_t *) iov, iovcnt); - - if (n == NXT_AGAIN) { - nxt_set_errno(NXT_EAGAIN); - return -1; - } - - return n; -} - -#endif - - -static void -nxt_gnutls_conn_handshake(nxt_thread_t *thr, void *obj, void *data) -{ - int err; - nxt_int_t ret; - nxt_event_conn_t *c; - nxt_gnutls_conn_t *ssltls; - - c = obj; - ssltls = c->u.ssltls; - - nxt_log_debug(thr->log, "gnutls conn handshake: %d", ssltls->times); - - /* "ssltls->times == 1" is suitable to run gnutls_handshake() in job. */ - - err = gnutls_handshake(ssltls->session); - - nxt_thread_time_debug_update(thr); - - nxt_log_debug(thr->log, "gnutls_handshake(): %d", err); - - if (err == GNUTLS_E_SUCCESS) { - nxt_gnutls_conn_io_read(thr, c, data); - return; - } - - ret = nxt_gnutls_conn_test_error(thr, c, err, nxt_gnutls_conn_handshake); - - if (ret == NXT_ERROR) { - nxt_gnutls_conn_log_error(c, err, "gnutls_handshake() failed"); - - nxt_event_conn_io_handle(thr, c->read_work_queue, - c->read_state->error_handler, c, data); - - } else if (err == GNUTLS_E_AGAIN - && ssltls->times < 2 - && gnutls_record_get_direction(ssltls->session) == 0) - { - ssltls->times++; - } -} - - -static void -nxt_gnutls_conn_io_read(nxt_thread_t *thr, void *obj, void *data) -{ - ssize_t n; - nxt_buf_t *b; - nxt_int_t ret; - nxt_event_conn_t *c; - nxt_gnutls_conn_t *ssltls; - nxt_work_handler_t handler; - - c = obj; - - nxt_log_debug(thr->log, "gnutls conn read"); - - handler = c->read_state->ready_handler; - b = c->read; - - /* b == NULL is used to test descriptor readiness. */ - - if (b != NULL) { - ssltls = c->u.ssltls; - - n = gnutls_record_recv(ssltls->session, b->mem.free, - b->mem.end - b->mem.free); - - nxt_log_debug(thr->log, "gnutls_record_recv(%d, %p, %uz): %z", - c->socket.fd, b->mem.free, b->mem.end - b->mem.free, n); - - if (n > 0) { - /* c->socket.read_ready is kept. */ - b->mem.free += n; - handler = c->read_state->ready_handler; - - } else if (n == 0) { - handler = c->read_state->close_handler; - - } else { - ret = nxt_gnutls_conn_test_error(thr, c, n, - nxt_gnutls_conn_io_read); - - if (nxt_fast_path(ret != NXT_ERROR)) { - return; - } - - nxt_gnutls_conn_log_error(c, n, - "gnutls_record_recv(%d, %p, %uz): failed", - c->socket.fd, b->mem.free, - b->mem.end - b->mem.free); - - handler = c->read_state->error_handler; - } - } - - nxt_event_conn_io_handle(thr, c->read_work_queue, handler, c, data); -} - - -static ssize_t -nxt_gnutls_conn_io_write_chunk(nxt_thread_t *thr, nxt_event_conn_t *c, - nxt_buf_t *b, size_t limit) -{ - nxt_gnutls_conn_t *ssltls; - - nxt_log_debug(thr->log, "gnutls conn write chunk"); - - ssltls = c->u.ssltls; - - return nxt_sendbuf_copy_coalesce(c, &ssltls->buffer, b, limit); -} - - -static ssize_t -nxt_gnutls_conn_io_send(nxt_event_conn_t *c, void *buf, size_t size) -{ - ssize_t n; - nxt_int_t ret; - nxt_gnutls_conn_t *ssltls; - - ssltls = c->u.ssltls; - - n = gnutls_record_send(ssltls->session, buf, size); - - nxt_log_debug(c->socket.log, "gnutls_record_send(%d, %p, %uz): %z", - c->socket.fd, buf, size, n); - - if (n > 0) { - return n; - } - - ret = nxt_gnutls_conn_test_error(nxt_thread(), c, n, - nxt_event_conn_io_write); - - if (nxt_slow_path(ret == NXT_ERROR)) { - nxt_gnutls_conn_log_error(c, n, - "gnutls_record_send(%d, %p, %uz): failed", - c->socket.fd, buf, size); - } - - return ret; -} - - -static void -nxt_gnutls_conn_io_shutdown(nxt_thread_t *thr, void *obj, void *data) -{ - int err; - nxt_int_t ret; - nxt_event_conn_t *c; - nxt_gnutls_conn_t *ssltls; - nxt_work_handler_t handler; - gnutls_close_request_t how; - - c = obj; - ssltls = c->u.ssltls; - - if (ssltls->session == NULL || ssltls->no_shutdown) { - handler = c->write_state->close_handler; - goto done; - } - - nxt_log_debug(c->socket.log, "gnutls conn shutdown"); - - if (c->socket.timedout || c->socket.error != 0) { - how = GNUTLS_SHUT_WR; - - } else if (c->socket.closed) { - how = GNUTLS_SHUT_RDWR; - - } else { - how = GNUTLS_SHUT_RDWR; - } - - err = gnutls_bye(ssltls->session, how); - - nxt_log_debug(c->socket.log, "gnutls_bye(%d, %d): %d", - c->socket.fd, how, err); - - if (err == GNUTLS_E_SUCCESS) { - handler = c->write_state->close_handler; - - } else { - ret = nxt_gnutls_conn_test_error(thr, c, err, - nxt_gnutls_conn_io_shutdown); - - if (ret != NXT_ERROR) { /* ret == NXT_AGAIN */ - c->socket.error_handler = c->read_state->error_handler; - nxt_event_timer_add(thr->engine, &c->read_timer, 5000); - return; - } - - nxt_gnutls_conn_log_error(c, err, "gnutls_bye(%d) failed", - c->socket.fd); - - handler = c->write_state->error_handler; - } - -done: - - nxt_event_conn_io_handle(thr, c->write_work_queue, handler, c, data); -} - - -static nxt_int_t -nxt_gnutls_conn_test_error(nxt_thread_t *thr, nxt_event_conn_t *c, ssize_t err, - nxt_work_handler_t handler) -{ - int ret; - nxt_gnutls_conn_t *ssltls; - - switch (err) { - - case GNUTLS_E_REHANDSHAKE: - case GNUTLS_E_AGAIN: - ssltls = c->u.ssltls; - ret = gnutls_record_get_direction(ssltls->session); - - nxt_log_debug(thr->log, "gnutls_record_get_direction(): %d", ret); - - if (ret == 0) { - /* A read direction. */ - - nxt_event_fd_block_write(thr->engine, &c->socket); - - c->socket.read_ready = 0; - c->socket.read_handler = handler; - - if (nxt_event_fd_is_disabled(c->socket.read)) { - nxt_event_fd_enable_read(thr->engine, &c->socket); - } - - } else { - /* A write direction. */ - - nxt_event_fd_block_read(thr->engine, &c->socket); - - c->socket.write_ready = 0; - c->socket.write_handler = handler; - - if (nxt_event_fd_is_disabled(c->socket.write)) { - nxt_event_fd_enable_write(thr->engine, &c->socket); - } - } - - return NXT_AGAIN; - - default: - c->socket.error = 1000; /* Nonexistent errno code. */ - return NXT_ERROR; - } -} - - -static void -nxt_gnutls_conn_log_error(nxt_event_conn_t *c, ssize_t err, - const char *fmt, ...) -{ - va_list args; - nxt_uint_t level; - u_char *p, msg[NXT_MAX_ERROR_STR]; - - level = nxt_gnutls_log_error_level(c, err); - - if (nxt_log_level_enough(c->socket.log, level)) { - - va_start(args, fmt); - p = nxt_vsprintf(msg, msg + sizeof(msg), fmt, args); - va_end(args); - - nxt_log_error(level, c->socket.log, "%*s (%d: %s)", - p - msg, msg, err, gnutls_strerror(err)); - } -} - - -static nxt_uint_t -nxt_gnutls_log_error_level(nxt_event_conn_t *c, ssize_t err) -{ - nxt_gnutls_conn_t *ssltls; - - switch (err) { - - case GNUTLS_E_UNKNOWN_CIPHER_SUITE: /* -21 */ - - /* Disable gnutls_bye(), because it returns GNUTLS_E_INTERNAL_ERROR. */ - ssltls = c->u.ssltls; - ssltls->no_shutdown = 1; - - /* Fall through. */ - - case GNUTLS_E_UNEXPECTED_PACKET_LENGTH: /* -9 */ - c->socket.error = 1000; /* Nonexistent errno code. */ - break; - - default: - return NXT_LOG_ALERT; - } - - return NXT_LOG_INFO; -} - - -static void -nxt_gnutls_log_error(nxt_uint_t level, nxt_log_t *log, int err, - const char *fmt, ...) -{ - va_list args; - u_char *p, msg[NXT_MAX_ERROR_STR]; - - va_start(args, fmt); - p = nxt_vsprintf(msg, msg + sizeof(msg), fmt, args); - va_end(args); - - nxt_log_error(level, log, "%*s (%d: %s)", - p - msg, msg, err, gnutls_strerror(err)); -} diff --git a/src/nxt_polarssl.c b/src/nxt_polarssl.c deleted file mode 100644 index 4a8144731..000000000 --- a/src/nxt_polarssl.c +++ /dev/null @@ -1,118 +0,0 @@ - -/* - * Copyright (C) NGINX, Inc. - * Copyright (C) Igor Sysoev - */ - -#include -#include -#include -#include -#include - - -typedef struct { - ssl_context ssl; - x509_cert certificate; - rsa_context key; -} nxt_polarssl_ctx_t; - - -static nxt_int_t nxt_polarssl_server_init(nxt_ssltls_conf_t *conf); -static void nxt_polarssl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c); -static void nxt_polarssl_log_error(nxt_uint_t level, nxt_log_t *log, int err, - const char *fmt, ...); - - -nxt_ssltls_lib_t nxt_polarssl_lib = { - nxt_polarssl_server_init, - NULL, -}; - - -static nxt_int_t -nxt_polarssl_server_init(nxt_ssltls_conf_t *conf) -{ - int n; - nxt_thread_t *thr; - nxt_polarssl_ctx_t *ctx; - - thr = nxt_thread(); - - /* TODO: mem_pool */ - - ctx = nxt_zalloc(sizeof(nxt_polarssl_ctx_t)); - if (ctx == NULL) { - return NXT_ERROR; - } - - n = ssl_init(&ctx->ssl); - if (n != 0) { - nxt_polarssl_log_error(NXT_LOG_ALERT, thr->log, n, "ssl_init() failed"); - return NXT_ERROR; - } - - ssl_set_endpoint(&ctx->ssl, SSL_IS_SERVER ); - - conf->ctx = ctx; - conf->conn_init = nxt_polarssl_conn_init; - - n = x509parse_crtfile(&ctx->certificate, conf->certificate); - if (n != 0) { - nxt_polarssl_log_error(NXT_LOG_ALERT, thr->log, n, - "x509parse_crt(\"%V\") failed", - &conf->certificate); - goto fail; - } - - rsa_init(&ctx->key, RSA_PKCS_V15, 0); - - n = x509parse_keyfile(&ctx->key, conf->certificate_key, NULL); - if (n != 0) { - nxt_polarssl_log_error(NXT_LOG_ALERT, thr->log, n, - "x509parse_key(\"%V\") failed", - &conf->certificate_key); - goto fail; - } - - ssl_set_own_cert(&ctx->ssl, &ctx->certificate, &ctx->key); - - /* TODO: ciphers */ - - /* TODO: ca_certificate */ - - return NXT_OK; - -fail: - - return NXT_ERROR; -} - - -static void -nxt_polarssl_conn_init(nxt_thread_t *thr, nxt_ssltls_conf_t *conf, - nxt_event_conn_t *c) -{ -} - - -static void -nxt_polarssl_log_error(nxt_uint_t level, nxt_log_t *log, int err, - const char *fmt, ...) -{ - va_list args; - u_char *p, *end, msg[NXT_MAX_ERROR_STR]; - - end = msg + NXT_MAX_ERROR_STR; - - va_start(args, fmt); - p = nxt_vsprintf(msg, end, fmt, args); - va_end(args); - - p = nxt_sprintf(p, end, " (%d: ", err); - - error_strerror(err, (char *) msg, p - msg); - - nxt_log_error(level, log, "%*s)", p - msg, msg); -} diff --git a/src/nxt_service.c b/src/nxt_service.c index cc9f8d894..33bc8f978 100644 --- a/src/nxt_service.c +++ b/src/nxt_service.c @@ -44,16 +44,6 @@ static const nxt_service_t nxt_services[] = { { "SSL/TLS", "openssl", &nxt_openssl_lib }, #endif -#if (NXT_HAVE_GNUTLS) - { "SSL/TLS", "GnuTLS", &nxt_gnutls_lib }, - { "SSL/TLS", "gnutls", &nxt_gnutls_lib }, -#endif - -#if (NXT_HAVE_CYASSL) - { "SSL/TLS", "CyaSSL", &nxt_cyassl_lib }, - { "SSL/TLS", "cyassl", &nxt_cyassl_lib }, -#endif - }; diff --git a/src/nxt_tls.h b/src/nxt_tls.h index 0667ade3c..0fb6cfce4 100644 --- a/src/nxt_tls.h +++ b/src/nxt_tls.h @@ -100,17 +100,4 @@ void nxt_cdecl nxt_openssl_log_error(nxt_task_t *task, nxt_uint_t level, u_char *nxt_openssl_copy_error(u_char *p, u_char *end); #endif -#if (NXT_HAVE_GNUTLS) -extern const nxt_tls_lib_t nxt_gnutls_lib; -#endif - -#if (NXT_HAVE_CYASSL) -extern const nxt_tls_lib_t nxt_cyassl_lib; -#endif - -#if (NXT_HAVE_POLARSSL) -extern const nxt_tls_lib_t nxt_polar_lib; -#endif - - #endif /* _NXT_TLS_H_INCLUDED_ */ From aa4de0292224ca8a0e4e5e582db632e160a2bb6c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 05:36:55 +0200 Subject: [PATCH 185/391] unitctl: stop reading JSON5 The configuration upload path stopped parsing its input, so it sends the file as it was written. JSON5 was the one format it still parsed, and it parsed it only to serialize the result straight back out as JSON. "edit" had already moved to JSON, so nothing else reached the parser. Refuse a ".json5" file by name instead, the way hjson and YAML are refused, and name a converter in the message. Keeping the InputFormat variant is what makes that refusal reachable: without it the file becomes InputFormat::Unknown, and execute.rs ends its format branch with panic!("Unknown input file type"). 140 crates to 138. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/Cargo.lock | 17 ------- tools/unitctl/unitctl/Cargo.toml | 1 - tools/unitctl/unitctl/src/inputfile.rs | 61 +++++++++++++++----------- tools/unitctl/unitctl/src/requests.rs | 4 +- tools/unitctl/unitctl/src/unitctl.rs | 2 +- 5 files changed, 38 insertions(+), 47 deletions(-) diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index a3116c9e3..a210dbec9 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -761,16 +761,6 @@ dependencies = [ "libc", ] -[[package]] -name = "json5" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733a844dbd6fef128e98cb4487b887cb55454d92cd9994b1bafe004fabbe670c" -dependencies = [ - "serde", - "ucd-trie", -] - [[package]] name = "libc" version = "0.2.186" @@ -1437,12 +1427,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "ucd-trie" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" - [[package]] name = "unicode-ident" version = "1.0.24" @@ -1502,7 +1486,6 @@ dependencies = [ "futures", "http-body-util", "hyper", - "json5", "rustls-pki-types", "serde", "serde_json", diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index 48549b47c..3c22fa1a9 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -19,7 +19,6 @@ tls = ["unit-client-rs/tls"] clap = { version = "4.4", features = ["default", "derive", "cargo"] } custom_error = "1.9" serde = "1.0" -json5 = "1.3.1" serde_json = { version = "1.0", optional = false } # The "std" feature is not on by default and pem::from_buf needs it. rustls-pki-types = { version = "1.15", features = ["std"] } diff --git a/tools/unitctl/unitctl/src/inputfile.rs b/tools/unitctl/unitctl/src/inputfile.rs index d63747e9c..ff9cd8fc5 100644 --- a/tools/unitctl/unitctl/src/inputfile.rs +++ b/tools/unitctl/unitctl/src/inputfile.rs @@ -5,7 +5,6 @@ use std::path::{Path, PathBuf}; use crate::known_size::KnownSize; use clap::ValueEnum; -use super::UnitSerializableMap; use super::UnitctlError; /// Input file data format @@ -146,39 +145,27 @@ impl InputFile { /// The media type and the bytes to PUT for a configuration input. /// - /// The type is the type of the body, not of the file: a JSON5 file is - /// parsed and sent as JSON, so it goes out as "application/json". - /// /// JSON is sent as it was written. unitctl does not parse it, so an /// operator's duplicate member reaches the server, which refuses it /// (src/nxt_conf.c:1616), and a number keeps the spelling the file used. /// Parsing here collapsed both silently. - /// - /// JSON5 is not JSON, so it is still parsed and re-serialized. That is - /// the one input whose bytes cannot go out unchanged. pub fn to_config_body(&self) -> Result<(String, KnownSize), UnitctlError> { let json = "application/json".to_string(); match self.format() { InputFormat::Json => Ok((json, self.try_into()?)), - InputFormat::Json5 => { - let mut reader: Box = self.try_into()?; - let mut json5_string = String::new(); - reader - .read_to_string(&mut json5_string) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?; - let parsed: UnitSerializableMap = json5::from_str(&json5_string) - .map_err(|e| UnitctlError::DeserializationError { message: e.to_string() })?; - let body = serde_json::to_string(&parsed) - .map_err(|e| UnitctlError::SerializationError { message: e.to_string() })?; - Ok((json, KnownSize::String(body))) - } - // Refuse hjson and YAML by name. Sending the file as it is would - // make the server report a syntax error on the first comment or - // unquoted key, and that error does not tell the user what to do. + // Refuse the formats Unit cannot read by name. Sending such a + // file as it is would make the server report a syntax error on + // the first comment or unquoted key, and that error does not tell + // the user what to do. InputFormat::Hjson => Err(UnitctlError::DeserializationError { message: "hjson is no longer supported: convert the file to JSON first".to_string(), }), + InputFormat::Json5 => Err(UnitctlError::DeserializationError { + message: "JSON5 is no longer supported: convert the file to JSON first, for example \ + with \"json5 -o config.json config.json5\"" + .to_string(), + }), InputFormat::Yaml => Err(UnitctlError::DeserializationError { message: "YAML is no longer supported: convert the file to JSON first, for example with \ \"yq -o=json\"" @@ -254,6 +241,7 @@ impl TryInto for &InputFile { #[cfg(test)] mod tests { use super::*; + use crate::UnitSerializableMap; #[test] fn can_parse_file_extensions() { @@ -355,10 +343,11 @@ mod tests { } /// The body Unit receives keeps member order. JSON is sent as it was - /// written, so it can only keep it. JSON5 is parsed and re-serialized, and - /// both ends of that have to keep it. + /// written, so it can only keep it. The check stays because the response + /// side still parses into `UnitSerializableMap`, which keeps order only + /// while `serde_json`'s "preserve_order" feature is on. #[test] - fn every_input_format_keeps_member_order() { + fn a_json_body_keeps_member_order() { let json = MEMBERS .iter() .map(|name| format!(" \"{}\": {{}}", name)) @@ -367,7 +356,6 @@ mod tests { let json = format!("{{\n{}\n}}\n", json); assert_eq!(members_of(&write_input(&json, ".json"), InputFormat::Json), MEMBERS); - assert_eq!(members_of(&write_input(&json, ".json5"), InputFormat::Json5), MEMBERS); } /// A JSON file is sent byte for byte. Parsing it here kept the last of two @@ -423,6 +411,27 @@ mod tests { ); } + /// JSON5 is refused by name too. unitctl parsed it until the PUT path + /// stopped parsing anything; keeping the variant means a ".json5" file + /// gets told what to do instead of reaching the JSON parser. + #[test] + fn a_json5_input_is_refused_with_a_message_about_json5() { + let file = write_input("{\n // a comment\n routes: [],\n}\n", ".json5"); + let input = InputFile::from(file.path()); + + assert!(input.is_config(), ".json5 must stay a config format"); + + let Err(error) = input.to_config_body() else { + panic!("JSON5 must be refused"); + }; + + assert!( + error.to_string().contains("JSON5 is no longer supported"), + "unexpected message: {}", + error + ); + } + /// YAML is refused by name for the same reason, and the ".yaml" extension /// still has to reach that refusal. Were the variant dropped instead, the /// file would become `InputFormat::Unknown` and `execute` would abort on diff --git a/tools/unitctl/unitctl/src/requests.rs b/tools/unitctl/unitctl/src/requests.rs index 28a382c8a..116df0a0b 100644 --- a/tools/unitctl/unitctl/src/requests.rs +++ b/tools/unitctl/unitctl/src/requests.rs @@ -17,8 +17,8 @@ use unit_client_rs::unit_client::UnitClientError; /// Send a configuration file to the unit server. /// -/// The bytes go out as they were written, for every format but JSON5. unitctl -/// used to parse the file into a `serde_json::Map` and re-serialize it. That +/// The bytes go out as they were written. unitctl used to parse the file +/// into a `serde_json::Map` and re-serialize it. That /// dropped an operator's duplicate member before the server could refuse it /// (src/nxt_conf.c:1616) and re-spelled every number. The server validates: /// UTF-8 (src/nxt_conf_validation.c:1840), duplicates, and the schema. diff --git a/tools/unitctl/unitctl/src/unitctl.rs b/tools/unitctl/unitctl/src/unitctl.rs index 37b4e8039..ab0f6d48e 100644 --- a/tools/unitctl/unitctl/src/unitctl.rs +++ b/tools/unitctl/unitctl/src/unitctl.rs @@ -84,7 +84,7 @@ pub(crate) enum Commands { global = true, short = 'f', long = "file", - help = "Input file (json, json5, pem) to send to unit when applicable use - for stdin" + help = "Input file (json, pem) to send to unit when applicable use - for stdin" )] input_file: Option, From 335ac60eb46eaed8b1ff5b21cff165fadfcd356a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 05:36:55 +0200 Subject: [PATCH 186/391] docs(changes): record that unitctl no longer reads JSON5 A ".json5" file that worked yesterday is refused today, so it belongs in the release notes with the conversion command. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 12 +++++++++--- docs/changes.xml | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index ebf525751..cdb789f9d 100644 --- a/CHANGES +++ b/CHANGES @@ -40,10 +40,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 parsed the file and re-serialized it before, which kept the last of two object members with the same name and re-spelled numbers, so the server never saw what the operator wrote. Unit now reports the duplicate - itself, with the line and column of the file. JSON5 input is still - parsed, because it is not JSON. A file that is not valid UTF-8 is no - longer refused by unitctl; Unit refuses it and names the member. The + itself, with the line and column of the file. A file that is not + valid UTF-8 is no longer refused by unitctl; Unit refuses it and names + the member. The "edit" command no longer accepts JSON5 comments in the temporary file. + *) Change: unitctl no longer reads JSON5. A ".json5" file is refused by + name, with a message saying to convert it first; "json5 -o config.json + config.json5" does that, and "jq" reformats ordinary JSON. Nothing + parsed JSON5 after the configuration upload path stopped parsing its + input, so this removes a parser that only the refusal path still + reached, and two crates with it. *) Feature: conditional requests for static files. A "share" action now answers a GET or HEAD carrying "If-None-Match" or "If-Modified-Since" diff --git a/docs/changes.xml b/docs/changes.xml index d4e1f8bb2..6ae38fdde 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -333,9 +333,8 @@ unitctl sends a JSON configuration file as it was written. It parsed the file and re-serialized it before, which kept the last of two object members with the same name and re-spelled numbers, so the server never saw what the operator wrote. Unit now reports the duplicate itself, with the line and column of the -file. JSON5 input is still parsed, because it is not JSON. A file that is not -valid UTF-8 is no longer refused by unitctl; Unit refuses it and names the -member. The "edit" command no longer accepts JSON5 comments in the temporary +file. A file that is not valid UTF-8 is no longer refused by unitctl; Unit +refuses it and names the member. The "edit" command no longer accepts JSON5 comments in the temporary file. @@ -355,6 +354,16 @@ unaffected; none of these files were ever part of a default build. + + +unitctl no longer reads JSON5. A ".json5" file is refused by name, with a +message saying to convert it first; "json5 -o config.json config.json5" does +that, and "jq" reformats ordinary JSON. Nothing parsed JSON5 after the +configuration upload path stopped parsing its input, so this removes a parser +that only the refusal path still reached, and two crates with it. + + + From 4cbc372fe6f4cde25e62acdec5955912b89aac1f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 04:59:59 +0200 Subject: [PATCH 187/391] changelog: correct the releases the 406 descriptor leak shipped in The entry said the leak was present "since compression was added: 1.35.4, 1.35.5, 1.36.0 and 1.36.1". Compression was added in 1.35.0 -- CHANGES lists "Feature: HTTP compression" under 1.35.0, 03 Sep 2025 -- so four more releases carry it. Checked at each tag rather than read off a diff. git show :src/nxt_http_static.c has the NXT_HTTP_NOT_ACCEPTABLE exit returning with no adjacent close in 1.35.0, v1.35.1, 1.35.2 and 1.35.3 as well, while the bodyless exits beside it close first; in 1.35.0 the open is at lines 402-440 and that exit is at 584, and at 584-586 in v1.35.1. 1.34.x cannot be affected, having no src/nxt_http_compression.c at all. 1.35.1 is tagged v1.35.1, with a "v" prefix no other release tag uses, so a tag listing filtered on "1.35.*" does not show it. It is a real release: CHANGES records it at 03 Apr 2026 as the fork point from nginx/unit, and it ships the compression code unchanged from 1.35.0. This matters beyond the changelog: an advisory for this leak is being prepared from the same range, and an operator on 1.35.0 or 1.35.1 reading the released notes would conclude they were never exposed. Co-Authored-By: Claude Opus 5 --- CHANGES | 5 +++-- docs/changes.xml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index cdb789f9d..bf80ace19 100644 --- a/CHANGES +++ b/CHANGES @@ -77,8 +77,9 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 the file it had opened, so a single unauthenticated request carrying "Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, and repeating it exhausted the table, stopping every application - behind that router. Present since compression was added: 1.35.4, - 1.35.5, 1.36.0 and 1.36.1 are all affected. + behind that router. Present since compression was added in 1.35.0: + 1.35.0, 1.35.1, 1.35.2, 1.35.3, 1.35.4, 1.35.5, 1.36.0 and 1.36.1 + are all affected. *) Security: a file descriptor sent to a port is no longer leaked when the message is dispatched and nothing takes ownership of it. An diff --git a/docs/changes.xml b/docs/changes.xml index 6ae38fdde..f80057ea8 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -77,8 +77,8 @@ configured answered "406 Not Acceptable" without closing the file it had opened, so a single unauthenticated request carrying "Accept-Encoding: identity;q=0, *;q=0" cost the router a descriptor, and repeating it exhausted the table, stopping every application behind that -router. Present since compression was added: 1.35.4, 1.35.5, 1.36.0 and -1.36.1 are all affected. +router. Present since compression was added in 1.35.0: 1.35.0, 1.35.1, +1.35.2, 1.35.3, 1.35.4, 1.35.5, 1.36.0 and 1.36.1 are all affected. From d2dc364497d94ff13b46f35bacad3b8d0f314bd1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 06:01:48 +0200 Subject: [PATCH 188/391] test(static): cover the Vary merge branch by branch nxt_http_comp_merge_vary() carries the fixes from four review rounds and was covered by two assertions: a gzip response has Vary, an identity response has it too. Both only ever reach the tail that creates the field. Every merge branch -- wildcard, empty value, already-listed token, trailing separator, and the re-assert after "response_headers" -- was verified by hand in PR comments and left untested, including the last one, a P1 that silently reintroduced the bug #360 exists to fix. Driven through "response_headers" rather than an application. That turned out to matter: it passes the configured value through verbatim, where PHP normalises trailing whitespace before libunit sees it, so it is the one source measured to deliver an untrimmed value. Each case observes its own defect. With the re-assert disabled six fail, including "Vary: Origin" emitting no Accept-Encoding at all; with the wildcard trim removed, "* " emits "*, Accept-Encoding", narrowing a header that varied on everything. Swapping the token scan's nxt_strncasecmp() for nxt_strncmp() fails the two case-varying cases and nothing else. Removal is covered by asserting the emitted value is a str: setting a field to null is f->skip = 1 rather than an erasure, and the scan filters on !f->skip, so a removed Vary is invisible and the tail adds a second field -- the test client collapses one occurrence to a str and two to a list, so this fails if the skipped field is ever serialised. Each mutation above was made and rebuilt, not reasoned about. Two limits are recorded in the file rather than papered over. A configured key of "vary" does not pin the merge's own field-name comparison: making it case-sensitive leaves the test passing, because "response_headers" matches the generated field case-insensitively itself, so the field reaching the merge is named "Vary" whatever the configuration wrote. And every test here drives the merge through "response_headers", so the path an application-authored Vary takes -- nxt_router_response_ready_handler(), which any CORS application emitting "Vary: Origin" reaches -- is uncovered, and covering it needs a language-module test. The trim's wider rationale rests on that same path, so it stays marked unverified. 25 passed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015j5NXxjo8hdYc99A8Db7tG --- test/test_static_compression.py | 186 ++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/test/test_static_compression.py b/test/test_static_compression.py index 9ec5b1f28..e0356f864 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -129,3 +129,189 @@ def get(**headers): assert ( resp['headers']['Vary'] == 'Accept-Encoding' ), 'Vary on the identity response too' + + +@pytest.mark.parametrize( + ('configured', 'expected'), + [ + # No "Vary" of its own: the generated field stands unchanged. + (None, 'Accept-Encoding'), + # An empty value has no tokens to append to, so it is replaced + # rather than appended to -- otherwise ", Accept-Encoding". + ('', 'Accept-Encoding'), + # A different header still needs the coding added: the response + # varies on both. An operator writing "Origin" is adding CORS and + # is almost never aware Unit generates the field at all. + ('Origin', 'Origin, Accept-Encoding'), + ('a, b, c', 'a, b, c, Accept-Encoding'), + # Already named: left exactly as written, and compared per token so + # that "X-Accept-Encoding" does not count as a match. + ('Accept-Encoding', 'Accept-Encoding'), + ('a, Accept-Encoding', 'a, Accept-Encoding'), + ('X-Accept-Encoding', 'X-Accept-Encoding, Accept-Encoding'), + # The token scan compares case-insensitively, as a field name must + # be. With nxt_strncasecmp() swapped for nxt_strncmp() every other + # case here still passes; this one appends and fails. + ('accept-encoding', 'accept-encoding'), + ('a, ACCEPT-ENCODING', 'a, ACCEPT-ENCODING'), + # "*" varies on everything already; adding to it would say less. + ('*', '*'), + # A trailing separator is appended after the last real token, not + # after the comma: "Origin,," is legal but pointless. + ('Origin,', 'Origin, Accept-Encoding'), + ('Origin, ', 'Origin, Accept-Encoding'), + # Optional whitespace is trimmed before the comparisons, and the + # value itself is left alone. "response_headers" is the one source + # measured to deliver an untrimmed value: it passes the configured + # string through verbatim, where PHP normalises trailing whitespace + # before libunit sees it. Whether any other language module also + # delivers one is untested, so treat the trim's wider rationale as + # unverified and this case as the reason it stays. + # + # Without the trim around the wildcard test, "* " appends and emits + # "*, Accept-Encoding", narrowing a header that varied on + # everything. The last case is trimmed per token instead, by the + # "already listed?" scan. + # + # The leading-space case also depends on the test client preserving + # leading OWS in a field value, which some parsers strip; it is + # asserting Unit's behaviour through a client that happens not to. + ('* ', '* '), + (' * ', ' * '), + ('Accept-Encoding ', 'Accept-Encoding '), + ], +) +def test_static_compression_vary_merge(temp_dir, configured, expected): + # "response_headers" runs after the field is generated and replaces or + # removes a field it names outright. A response chosen by negotiation + # varies on Accept-Encoding whatever an operator wrote there, so the + # merge is re-asserted afterwards; without that, a "Vary: Origin" in the + # configuration silently drops the coding from every shared cache key. + action = {"share": f'{temp_dir}/assets$uri'} + + if configured is not None: + action["response_headers"] = {"Vary": configured} + + assert 'success' in client.conf(action, 'routes/0/action'), 'configure' + + resp = client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'gzip', + }, + ) + + assert resp['status'] == 200, 'status 200' + assert resp['headers']['Content-Encoding'] == 'gzip', 'compressed' + assert resp['headers']['Vary'] == expected, 'merged Vary' + + +@pytest.mark.parametrize('coding', ['gzip', None]) +def test_static_compression_vary_merge_removed(temp_dir, coding): + # "response_headers" can remove a field outright by setting it to null, + # and removal is f->skip = 1 rather than an erasure. The merge scan + # filters on !f->skip, so a removed Vary is invisible to it: "vary" stays + # NULL and control reaches the tail that adds a fresh field. The response + # therefore holds two Vary fields, one skipped and one new. + # + # Assert the emitted header is a single string, not a list. The test + # client collapses one occurrence to a str and two to a list, so this + # fails if the skipped field is ever serialised -- a class of bug this + # code has hit before, where a skipped response field's slot is reused. + assert 'success' in client.conf( + { + "share": f'{temp_dir}/assets$uri', + "response_headers": {"Vary": None}, + }, + 'routes/0/action', + ), 'configure' + + headers = {'Host': 'localhost', 'Connection': 'close'} + + if coding is not None: + headers['Accept-Encoding'] = coding + + resp = client.get(url='/big.css', headers=headers) + + assert resp['status'] == 200, 'status 200' + assert resp['headers']['Vary'] == 'Accept-Encoding', 're-added' + assert isinstance(resp['headers']['Vary'], str), 'exactly one Vary' + + +def test_static_compression_vary_merge_field_name_case(temp_dir): + # A configured key of "vary" is still merged rather than emitted beside + # the generated field. + # + # Measured limit: this does NOT pin the nxt_strncasecmp(f->name, "Vary") + # comparison inside nxt_http_comp_merge_vary(). Making that comparison + # case-sensitive leaves this test passing, because "response_headers" + # matches the generated field case-insensitively itself and replaces its + # value, so the field reaching the merge is still named "Vary" whatever + # the configuration wrote. Pinning the merge's own field-name comparison + # needs a header authored by an application, which no test here can do -- + # see the note on the uncovered application path below. + assert 'success' in client.conf( + { + "share": f'{temp_dir}/assets$uri', + "response_headers": {"vary": "Origin"}, + }, + 'routes/0/action', + ), 'configure' + + resp = client.get( + url='/big.css', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'gzip', + }, + ) + + assert resp['status'] == 200, 'status 200' + assert resp['headers']['Vary'] == 'Origin, Accept-Encoding', 'merged' + + +@pytest.mark.parametrize( + ('configured', 'expected'), + [ + ('Origin', 'Origin, Accept-Encoding'), + ('*', '*'), + ('Accept-Encoding', 'Accept-Encoding'), + ], +) +def test_static_compression_vary_merge_identity(temp_dir, configured, expected): + # r->resp.vary_accept_encoding is set in nxt_http_comp_check_acceptable() + # before any coding is chosen, so the merge and the re-assert both run for + # an identity response too. The identity response is precisely the one a + # cache must not reuse for a gzip-capable client, so a merge that only + # worked on the coded path would leave the hole open from the other side. + assert 'success' in client.conf( + { + "share": f'{temp_dir}/assets$uri', + "response_headers": {"Vary": configured}, + }, + 'routes/0/action', + ), 'configure' + + resp = client.get( + url='/big.css', + headers={'Host': 'localhost', 'Connection': 'close'}, + ) + + assert resp['status'] == 200, 'status 200' + assert 'Content-Encoding' not in resp['headers'], 'not compressed' + assert resp['headers']['Vary'] == expected, 'merged on identity too' + + +# Uncovered by design, recorded so the gap is not mistaken for coverage: +# every test above drives the merge through "response_headers" against a +# "share", so the pre-existing Vary is always one Unit itself generated and +# response_headers then replaced. A Vary authored by an *application* reaches +# nxt_http_comp_merge_vary() by the other caller, +# nxt_router_response_ready_handler(), which copies the app's fields into +# r->resp before nxt_http_comp_check_acceptable() runs. Any CORS application +# emitting "Vary: Origin" takes that path, and nothing here exercises it -- +# including the merge's own case-insensitive field-name match. Covering it +# needs a language-module test, not a static one. From 5ee639071b807f519191c0aabf5fe8a4a1acd9bd Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 06:45:25 +0200 Subject: [PATCH 189/391] test: keep a truncated log in the type it arrived as "pytest --detailed" fails with AttributeError on any request whose logged bytes pass the 1024-byte limit. Measured on test_static.py: 2 failed, 53 passed before, 55 passed after. The cause is not the decode that raises. log_truncate() appended its "(...logged N of M bytes)" note with an f-string, and an f-string on bytes formats the repr: b'GET /...' became the 9-character-longer string "b'GET /...'". The caller then called .decode() on a str. The appendix.encode() branch right above it could never have any effect, which is the tell. Concatenate instead, and keep each branch in its own type. log_out() then decodes only what is still bytes, and keeps the UnicodeEncodeError guard for the print itself, which is what that guard was for. Nothing in CI passes --detailed, so this rotted unseen. Adding such a job is worth doing separately; it would have to run a suite that reaches the truncation path to catch this again. Co-Authored-By: Claude Opus 5 (1M context) --- test/unit/http.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/unit/http.py b/test/unit/http.py index e449c8b54..84af79d57 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -131,10 +131,14 @@ def log_out(self, log, encoding): if option.detailed: print('>>>') log = self.log_truncate(log) + + if isinstance(log, bytes): + log = log.decode(encoding, 'ignore') + try: - print(log.decode(encoding, 'ignore')) - except UnicodeEncodeError: print(log) + except UnicodeEncodeError: + print(log.encode()) def log_in(self, log): if option.detailed: @@ -148,13 +152,15 @@ def log_in(self, log): def log_truncate(self, log, limit=1024): len_log = len(log) if len_log > limit: - log = log[:limit] appendix = f'(...logged {limit} of {len_log} bytes)' + # Concatenate, do not format: an f-string turns bytes into its + # repr, so the result was a str holding "b'...'" and the caller + # that decodes it got an AttributeError instead of a log. if isinstance(log, bytes): - appendix = appendix.encode() - - log = f'{log}{appendix}' + log = log[:limit] + appendix.encode() + else: + log = log[:limit] + appendix return log From 2ab6eb71bd6c089620415430ff6fe6321650201f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 06:45:48 +0200 Subject: [PATCH 190/391] ci(fuzzing): run on a change to the UTF-8 decoder nxt_basic_fuzz.c calls nxt_utf8_decode() and nxt_utf8_length() (fuzzing/nxt_basic_fuzz.c:191,195), and nxt_basic_fuzz.o is linked into fuzz_basic (auto/fuzzing:32). The decoder is fuzzed, but a pull request that changes it does not run this job: src/nxt_utf8* was missing from the path filter. Add it. The other half of this, making the target loop over its input instead of decoding one sequence, needs the decoder to report how much it consumed; that is tracked in #311 and does not belong here. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/fuzzing.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index 6d37ab1ee..a0863aad0 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -8,6 +8,9 @@ on: - 'src/nxt_http*' - 'src/nxt_h1proto*' - 'src/nxt_conf*' + # nxt_basic_fuzz.c drives nxt_utf8_decode() and nxt_utf8_length(), so a + # change to the decoder is a change to what this job fuzzes. + - 'src/nxt_utf8*' # nxt_http_controller_fuzz.c #includes nxt_controller.c, so a change # there is a change to what this job fuzzes. - 'src/nxt_controller*' From 77a4528ed317d227fc323940b5fb7f460f04e35b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 06:49:12 +0200 Subject: [PATCH 191/391] unitctl: send a JavaScript module instead of aborting "unitctl execute -m PUT -p /js_modules/mod.js -f mod.js" panicked: thread 'main' panicked at unitctl/src/cmd/execute.rs:80:17: Unknown input file type exit 101, a backtrace, and no request sent. The dispatch tested is_config() and is_pem_bundle() and nothing else, though is_javascript() has been there all along and "import" uses it. The GET direction already special-cases /js_modules/*.js, so reading a module worked while writing one aborted. Add the branch. It sends the bytes with send_body_deserialize_response(), which is what "import" does, so the request is the same one either command produces: PUT, "Content-Type: application/javascript", body verbatim. Refuse an unplaceable file rather than aborting, in both commands. A file name the operator typed is not a reason to abort; UnknownInputFileType already exists and names the file. In "import" the panic sat behind a check that already refused InputFormat::Unknown, so it was unreachable -- turning that From into a TryFrom keeps it that way by construction. Checked against a mock control socket: PUT /js_modules/mod.js, content-type application/javascript, body unchanged. The old binary on the same file exits 101. Co-Authored-By: Claude Opus 5 (1M context) --- tools/unitctl/unitctl/src/cmd/execute.rs | 12 ++++- tools/unitctl/unitctl/src/cmd/import.rs | 62 +++++++++++++++++++++--- tools/unitctl/unitctl/src/inputfile.rs | 5 +- 3 files changed, 68 insertions(+), 11 deletions(-) diff --git a/tools/unitctl/unitctl/src/cmd/execute.rs b/tools/unitctl/unitctl/src/cmd/execute.rs index 4e4df4d2f..02438310a 100644 --- a/tools/unitctl/unitctl/src/cmd/execute.rs +++ b/tools/unitctl/unitctl/src/cmd/execute.rs @@ -75,9 +75,17 @@ async fn send_and_deserialize( // TLS certificate data } else if input_file.is_pem_bundle() { send_and_validate_pem_data_deserialize_response(&client, method.as_str(), path, &input_file).await - // This is unknown data + // A JS module: the bytes go up as they are, the way "import" sends them + } else if input_file.is_javascript() { + send_body_deserialize_response(&client, method.as_str(), path, Some(&input_file)).await + // A file this command cannot send anywhere } else { - panic!("Unknown input file type") + Err(UnitctlError::UnknownInputFileType { + path: input_file.to_path().map_or_else( + |_| "-".to_string(), + |path| path.to_string_lossy().into_owned(), + ), + }) } } // A none value for an input file can be considered a request to send an empty body diff --git a/tools/unitctl/unitctl/src/cmd/import.rs b/tools/unitctl/unitctl/src/cmd/import.rs index e4726a83a..7106dc3be 100644 --- a/tools/unitctl/unitctl/src/cmd/import.rs +++ b/tools/unitctl/unitctl/src/cmd/import.rs @@ -12,16 +12,22 @@ enum UploadFormat { Javascript, } -impl From<&InputFile> for UploadFormat { - fn from(input_file: &InputFile) -> Self { +impl TryFrom<&InputFile> for UploadFormat { + type Error = UnitctlError; + + /// A file this command cannot place is refused, not fatal. This used to + /// panic on a format that reached it without a branch here. + fn try_from(input_file: &InputFile) -> Result { if input_file.is_config() { - UploadFormat::Config + Ok(UploadFormat::Config) } else if input_file.is_pem_bundle() { - UploadFormat::PemBundle + Ok(UploadFormat::PemBundle) } else if input_file.is_javascript() { - UploadFormat::Javascript + Ok(UploadFormat::Javascript) } else { - panic!("Unknown input file type"); + Err(UnitctlError::UnknownInputFileType { + path: input_file.to_path()?.to_string_lossy().into(), + }) } } } @@ -88,7 +94,7 @@ async fn process_entry(entry: DirEntry, client: &UnitClient) -> Result<(), Unitc path: input_file.to_path()?.to_string_lossy().into(), }); } - let upload_format = UploadFormat::from(&input_file); + let upload_format = UploadFormat::try_from(&input_file)?; let upload_path = upload_format.upload_path(entry.path()); // We can't overwrite JS or PEM files, so we delete them first @@ -132,3 +138,45 @@ async fn process_entry(entry: DirEntry, client: &UnitClient) -> Result<(), Unitc } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn input(name: &str) -> InputFile { + InputFile::from(Path::new(name)) + } + + /// Each format this command can place has its own destination, and the + /// ".js" case is the one `execute` used to abort on. + #[test] + fn every_known_format_has_an_upload_path() { + let cases = [ + ("conf.json", "/config"), + ("bundle.pem", "/certificates/bundle.pem"), + ("mod.js", "/js_modules/mod.js"), + ]; + + for (name, expected) in cases { + let file = input(name); + let format = UploadFormat::try_from(&file).expect("a known format"); + + assert_eq!(format.upload_path(Path::new(name)), expected); + } + } + + /// A file this command cannot place is refused by name. It used to + /// panic, which gave the operator a backtrace instead of the file name. + #[test] + fn an_unplaceable_file_is_refused_not_fatal() { + let Err(error) = UploadFormat::try_from(&input("notes.txt")) else { + panic!("an unplaceable file must be refused"); + }; + + assert!( + matches!(error, UnitctlError::UnknownInputFileType { .. }), + "unexpected error: {}", + error + ); + } +} diff --git a/tools/unitctl/unitctl/src/inputfile.rs b/tools/unitctl/unitctl/src/inputfile.rs index ff9cd8fc5..e0cffaa50 100644 --- a/tools/unitctl/unitctl/src/inputfile.rs +++ b/tools/unitctl/unitctl/src/inputfile.rs @@ -434,8 +434,9 @@ mod tests { /// YAML is refused by name for the same reason, and the ".yaml" extension /// still has to reach that refusal. Were the variant dropped instead, the - /// file would become `InputFormat::Unknown` and `execute` would abort on - /// its `panic!("Unknown input file type")`. + /// file would become `InputFormat::Unknown`, which `execute` and `import` + /// report as an unknown input type -- the file name, but not what to do + /// about it. #[test] fn a_yaml_input_is_refused_with_a_message_about_yaml() { for suffix in [".yaml", ".yml"] { From b4e4110fb1256846a5401a378227d290adfeceae Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 03:57:28 +0200 Subject: [PATCH 192/391] port: take the message back when the port refuses it src/nxt_port.h states the contract: the port layer takes fd, fd2 and b only when it answers NXT_OK, and on any other answer they are still the caller's to close and complete. Eight callers hand a buffer over and do neither. The refusal is not hypothetical. nxt_port_msg_chk_insert() answers NXT_ERROR when it cannot allocate; the inline path answers NXT_ERROR for a first fragment that hit EAGAIN with no memory to hold it; and nxt_port_queue_send() answers NXT_AGAIN with nothing allocated at all, when the receiver has stopped draining a shared queue of 16384 items. That last one needs no memory pressure, and it is reachable on exactly the two callers that carry shared memory. Those two are the reason this matters. nxt_http_websocket.c passed its frame -- a chain of chunks from the application's outgoing segment -- and answered a failure with "// TODO: handle", so the chunks were never returned to the segment's free map. No pool teardown reclaims them: the segment loses that capacity for the life of the application, and enough lost frames starve it. nxt_router_req_headers_ack_handler() has the same shape, and worse odds of being noticed: it cuts the tail off msg_info.buf before the write, so nxt_router_msg_cancel() walks a chain that no longer reaches the tail and cannot complete it either. Both now return the chain, queued rather than run inline. The websocket site still reads r->ws_frame afterwards, and nxt_port_mmap_buf_completion() can post a SHM_ACK, so completing inline would re-enter the port layer in the middle of the caller -- which is why nxt_port_msg_drop() queues its own. Only the tail is completed in the router; the head stays with msg_info.buf, where the cancel path still finds it. The other six lose a plain buffer from an engine or port pool, which is never reclaimed while that engine or port lives: CHANGE_FILE and REMOVE_PID per peer, NEW_PORT per failed announce, SOCKET_UNLINK per unix socket, the WHOAMI reply in the main process, and the /status reply, which an operator can repeat. nxt_port_send_port() is the one that could not be fixed at its caller: its buffer never leaves the function, so the callers that check its answer have nothing to complete. It now cleans up before returning. Sites whose buffer is reclaimed by a pool teardown on the same failure path are deliberately untouched -- nxt_router.c's tmcf payloads with nxt_buf_dummy_completion, the controller's connection-pool buffers, the cert and script buffers whose nxt_mp_retain() is on the success path only. Completing those would free nothing at best, and unbalance a pool reference count at worst. src/test/nxt_port_fail_test.c drives nxt_port_send_port() with the allocation hook armed against a fixture whose engine pool holds nothing else, so nxt_mp_is_empty() answers the question directly: without the fix the leg reports "the buffer of a refused message was left in the caller's pool". Co-Authored-By: Claude Opus 5 --- CHANGES | 8 +++ docs/changes.xml | 11 ++++ src/nxt_http_websocket.c | 39 ++++++++++- src/nxt_main_process.c | 12 +++- src/nxt_port.c | 46 +++++++++++-- src/nxt_router.c | 47 ++++++++++++-- src/test/nxt_port_fail_test.c | 119 ++++++++++++++++++++++++++++++++++ 7 files changed, 268 insertions(+), 14 deletions(-) diff --git a/CHANGES b/CHANGES index bf80ace19..df2534a46 100644 --- a/CHANGES +++ b/CHANGES @@ -143,6 +143,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 prototype that was not coming, and the requests waiting on it were never failed. + *) Bugfix: eight places that handed a message to a port did not take it + back when the port refused it. The port layer accepts a buffer only + when it answers success, so on a refusal -- which happens when memory + runs out, or when the receiving process has stopped reading a full + shared queue -- the memory was lost. Two of them lost shared memory + that no cleanup reclaims, so an application could lose the capacity to + answer at all. + *) Bugfix: a port could stop sending after it ran out of memory. When a write had to be retried later, the port asked its own thread to watch the socket again, and that request needed memory of its own. If it diff --git a/docs/changes.xml b/docs/changes.xml index f80057ea8..7be9f285b 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -150,6 +150,17 @@ requests waiting on it were never failed. + + +eight places that handed a message to a port did not take it back when the port +refused it. The port layer accepts a buffer only when it answers success, so on +a refusal -- which happens when memory runs out, or when the receiving process +has stopped reading a full shared queue -- the memory was lost. Two of them +lost shared memory that no cleanup reclaims, so an application could lose the +capacity to answer at all. + + + a port could stop sending after it ran out of memory. When a write had to be diff --git a/src/nxt_http_websocket.c b/src/nxt_http_websocket.c index 39477e2dc..4b84d73eb 100644 --- a/src/nxt_http_websocket.c +++ b/src/nxt_http_websocket.c @@ -115,7 +115,44 @@ nxt_http_websocket_client(nxt_task_t *task, void *obj, void *data) req_rpc_data->stream, task->thread->engine->port->id, out); if (nxt_slow_path(res != NXT_OK)) { - // TODO: handle + /* + * The port layer did not take the chain, so it is still ours to + * return -- and nothing else will: these are chunks of the + * application's outgoing shared memory, which no pool teardown + * reclaims. A frame lost this way costs the segment capacity + * permanently, and the application starves once enough have gone. + * + * Queued rather than run here, as nxt_port_msg_drop() queues its + * own: nxt_port_mmap_buf_completion() can post a SHM_ACK, and + * re-entering the port layer from inside this call is worth + * avoiding even on the way out. + */ + + while (out != NULL) { + next = out->next; + out->next = NULL; + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + out->completion_handler, task, out, + out->parent); + + out = next; + } + + /* + * And fail the connection, the way the allocation failure above + * does. A non-OK answer here means the application port is gone or + * its shared queue is full, so this frame is not reaching the + * application at all; carrying on would leave the peer's stream + * short a frame it is never told about, which for a fragmented + * message is a protocol error the client cannot detect. This is + * what the "// TODO: handle" that used to stand here was asking + * for. + */ + + nxt_http_websocket_error_handler(task, r, r->proto.any); + + return; } b = r->ws_frame; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index e6d666255..0be5c4635 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -935,8 +935,16 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) buf->mem.free = nxt_cpymem(buf->mem.free, &pid, sizeof(nxt_pid_t)); - (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_READY_LAST, -1, - msg->port_msg.stream, 0, buf); + if (nxt_slow_path(nxt_port_socket_write(task, port, + NXT_PORT_MSG_RPC_READY_LAST, -1, + msg->port_msg.stream, 0, buf) + != NXT_OK)) + { + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + buf->completion_handler, task, buf, buf->parent); + } fail: diff --git a/src/nxt_port.c b/src/nxt_port.c index 940225785..1ae594628 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -462,6 +462,7 @@ nxt_int_t nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, nxt_port_t *new_port, uint32_t stream) { + nxt_int_t ret; nxt_buf_t *b; nxt_port_msg_new_port_t *msg; @@ -483,9 +484,24 @@ nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, nxt_port_t *new_port, msg->max_share = port->max_share; msg->type = new_port->type; - return nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, - new_port->pair[1], new_port->queue_fd, - stream, 0, b); + ret = nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, + new_port->pair[1], new_port->queue_fd, + stream, 0, b); + + if (nxt_slow_path(ret != NXT_OK)) { + /* + * Still ours, and nobody else can reach it: b never leaves this + * function, so a caller that sees the failure has nothing to + * complete. The descriptors are borrowed from new_port and stay + * with it, as they do on the success path -- NEW_PORT carries no + * NXT_PORT_MSG_CLOSE_FD. + */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + } + + return ret; } @@ -1056,8 +1072,15 @@ nxt_port_change_log_file(nxt_task_t *task, nxt_runtime_t *rt, nxt_uint_t slot, b->mem.free = nxt_cpymem(b->mem.free, &slot, sizeof(nxt_uint_t)); - (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_CHANGE_FILE, - fd, 0, 0, b); + if (nxt_slow_path(nxt_port_socket_write(task, port, + NXT_PORT_MSG_CHANGE_FILE, + fd, 0, 0, b) != NXT_OK)) + { + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + } } nxt_runtime_process_loop; } @@ -1216,8 +1239,17 @@ nxt_port_remove_notify_others(nxt_task_t *task, nxt_process_t *process) buf->mem.free = nxt_cpymem(buf->mem.free, &pid, sizeof(pid)); - nxt_port_socket_write(task, port, NXT_PORT_MSG_REMOVE_PID, -1, - process->stream, 0, buf); + if (nxt_slow_path(nxt_port_socket_write(task, port, + NXT_PORT_MSG_REMOVE_PID, -1, + process->stream, 0, buf) + != NXT_OK)) + { + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + buf->completion_handler, task, buf, + buf->parent); + } } nxt_runtime_process_loop; } diff --git a/src/nxt_router.c b/src/nxt_router.c index 0610c7206..1f242c0a7 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1734,7 +1734,20 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) fail: - nxt_port_socket_write(task, port, type, -1, msg->port_msg.stream, 0, b); + if (nxt_slow_path(nxt_port_socket_write(task, port, type, -1, + msg->port_msg.stream, 0, b) + != NXT_OK) + && b != NULL) + { + /* + * Still ours: the port layer takes the buffer only on NXT_OK, and + * this one lives in the reply port's pool, so it would sit there + * until the controller's port is released. + */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + } } @@ -5080,8 +5093,15 @@ nxt_router_listen_socket_release(nxt_task_t *task, nxt_socket_conf_t *skcf) rt = task->thread->runtime; main_port = rt->port_by_type[NXT_PROCESS_MAIN]; - (void) nxt_port_socket_write(task, main_port, NXT_PORT_MSG_SOCKET_UNLINK, - -1, 0, 0, b); + if (nxt_slow_path(nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_SOCKET_UNLINK, + -1, 0, 0, b) != NXT_OK)) + { + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + } out_free_ls: #endif @@ -5573,7 +5593,7 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, { int res; nxt_app_t *app; - nxt_buf_t *b; + nxt_buf_t *b, *next; nxt_bool_t start_process, unlinked; nxt_port_t *app_port, *main_app_port, *idle_port; nxt_queue_link_t *idle_lnk; @@ -5694,6 +5714,25 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, task->thread->engine->port->id, b); if (nxt_slow_path(res != NXT_OK)) { + /* + * This tail was cut from msg_info.buf above, so + * nxt_router_msg_cancel() walks a chain that no longer reaches + * it and nothing else completes it. Its buffers are chunks of + * shared memory that no pool teardown reclaims, so return them + * here. Queued rather than run inline, as the port layer does + * when it drops a message of its own. + */ + + while (b != NULL) { + next = b->next; + b->next = NULL; + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + b->completion_handler, task, b, b->parent); + + b = next; + } + nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); } } diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index 69e827d40..35d4c8325 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -27,6 +27,7 @@ static nxt_int_t nxt_port_fail_test_cross_engine_acquire(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_cross_engine_race(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_cross_engine_batch(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_cross_engine_rearm(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_caller_owns_buf(nxt_thread_t *thr); static void nxt_port_fail_test_racer(void *data); static nxt_int_t nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, nxt_work_t *item); @@ -96,6 +97,10 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_caller_owns_buf(thr) != NXT_OK) { + return NXT_ERROR; + } + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "port failure test passed"); @@ -1696,6 +1701,120 @@ nxt_port_fail_test_cross_engine_rearm(nxt_thread_t *thr) return ret; } + +/* + * A caller gets its buffer back when the port refuses the message. + * + * src/nxt_port.h says the port layer takes fd, fd2 and b only on NXT_OK, so on + * any other answer the caller still owns them. A caller that does not act on + * that leaks: nothing else completes the buffer, and for an engine pool that + * memory is never reclaimed while the process runs. + * + * nxt_port_send_port() stands in for the eight sites that hand a buffer over. + * It is the awkward one -- its buffer never leaves the function, so no caller + * of it could clean up even if it wanted to -- and it allocates from the + * engine pool, which makes nxt_mp_is_empty() a decisive check: the pool holds + * the buffer, and nothing else. + */ + +static nxt_int_t +nxt_port_fail_test_caller_owns_buf(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port, *new_port; + nxt_event_engine_t engine, *saved_engine; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + port = NULL; + new_port = NULL; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + /* The buffer comes from here, which is what makes the check decisive. */ + engine.mem_pool = mp; + + saved_engine = thr->engine; + thr->engine = &engine; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + new_port = nxt_port_fail_test_port(task); + if (nxt_slow_path(new_port == NULL)) { + goto done; + } + + if (nxt_slow_path(!nxt_mp_is_empty(mp))) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the fixture pool is not empty " + "before the write"); + goto done; + } + + nxt_port_test_msg_alloc_failures(1); + + ret = nxt_port_send_port(task, port, new_port, 0); + + nxt_port_test_msg_alloc_failures(0); + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the refused NEW_PORT answered " + "NXT_OK"); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_ERROR; + + /* The completion is queued, as nxt_port_msg_drop() queues its own. */ + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + if (nxt_slow_path(!nxt_mp_is_empty(mp))) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the buffer of a refused message was " + "left in the caller's pool"); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_port_test_msg_alloc_failures(0); + + if (new_port != NULL) { + nxt_port_use(task, new_port, -1); + } + + if (port != NULL) { + nxt_port_use(task, port, -1); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_mp_destroy(mp); + + return ret; +} + static nxt_int_t nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, nxt_work_t *item) { From 7e27eb02d13864497b8ae805792310cf16a74352 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 03:57:33 +0200 Subject: [PATCH 193/391] port: report a send that failed because the peer is gone nxt_port_socket_write2() answered NXT_OK for a message it had dropped when sendmsg() failed outright. The sender was told its message went out, so a caller that answers an RPC and then sends a fallback lost both -- the reply, and the fallback it cleared state for. Nothing of that message was consumed: nxt_socketpair_send() sent no bytes and no descriptor, and nxt_port_msg_close_fd() runs only after a successful send. So an inline first fragment is still whole and still the caller's, and NXT_ERROR is the honest answer. A fragment that has already gone out, and a queued message with no caller to tell, keep the old treatment: the port owns them and nxt_port_msg_drop() finishes them. This could not be done until the callers were ready for it. The previous commit made the eight that hand a buffer over take it back on a non-OK answer; without that, reporting here would have turned a lost message into a leak at every one of them, two of which lose shared memory that no pool teardown reclaims. A port whose peer is gone also arms nothing. One answer changed with the flip: the wake-up for a payload that reached the shared queue. That payload is the port's however the wake-up ends, so the answer stays NXT_OK, but a failed wake-up also records an owed queue marker and re-arms the write event to send it later. That is right for an EAGAIN, whose socket takes writes again. For a peer that is gone the marker can never be delivered, and the re-arm only brings the write handler back to fail the same send, again and again, on a port whose removal is already under way -- until the event fires on memory that has been freed. The application churn tests showed it as repeated EPIPE followed by pthread_mutex_lock() failing with EINVAL, and the sanitizer leg named it: heap-use-after-free in nxt_port_msg_first(), from nxt_port_write_handler(), on a port freed by nxt_port_mp_cleanup(). nxt_port_write_msgs() now reports that one exit through "peer_gone", and neither the marker nor the re-arm is taken when it is set. The three other exits that raise the error handler -- a short write, and two allocation failures after a fragment has gone out -- leave a working socket and keep re-arming on an owed marker exactly as before; the handler does not close a port, it drains the queued messages and drops one reference. nxt_router_start_timeout_test drove this path with a -1 descriptor and asserted that only the deadline bounded such a start. It now asserts the contract instead: the refusal fails the start inside the call, no deadline is needed, and -- the assertion that matters -- the work queue is empty afterwards, because a completion queued for a payload the caller has already freed is the double free this contract exists to prevent. nxt_port_fail_test gains the dead-peer half of the queued-payload leg it already had for EAGAIN: same enqueue, same NXT_OK, and then announce 0 with no re-arm. Without the fix the leg reports the owed marker. Fixes: #348 Co-Authored-By: Claude Opus 5 --- CHANGES | 24 ++- docs/changes.xml | 30 ++- src/nxt_application.c | 39 ++-- src/nxt_port_socket.c | 106 +++++++--- src/test/nxt_port_fail_test.c | 241 +++++++++++++++++++++++ src/test/nxt_router_start_timeout_test.c | 65 +++--- 6 files changed, 421 insertions(+), 84 deletions(-) diff --git a/CHANGES b/CHANGES index df2534a46..76b4a1a1f 100644 --- a/CHANGES +++ b/CHANGES @@ -143,13 +143,23 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 prototype that was not coming, and the requests waiting on it were never failed. - *) Bugfix: eight places that handed a message to a port did not take it - back when the port refused it. The port layer accepts a buffer only - when it answers success, so on a refusal -- which happens when memory - runs out, or when the receiving process has stopped reading a full - shared queue -- the memory was lost. Two of them lost shared memory - that no cleanup reclaims, so an application could lose the capacity to - answer at all. + *) Bugfix: a message that could not be sent because the receiving process + had died was reported to the sender as sent. A sender that answers a + request and then sends a fallback lost both, so an application could + wait for an answer that nothing would ever deliver. + + *) Bugfix: eight places whose memory nothing else reclaims handed a + message to a port and did not take it back when the port refused it. + The port layer accepts a buffer only when it answers success, so on a + refusal -- which happens when memory runs out, or when the receiving + process has stopped reading a full shared queue -- the memory was + lost. Two of them lost shared memory that no cleanup reclaims, so an + application could lose the capacity to answer at all. + + *) Change: a websocket connection is now closed when a frame cannot be + handed to the application. Such a frame was previously dropped in + silence, leaving the peer's stream short a frame it was never told + about -- which for a fragmented message the client cannot detect. *) Bugfix: a port could stop sending after it ran out of memory. When a write had to be retried later, the port asked its own thread to watch diff --git a/docs/changes.xml b/docs/changes.xml index 7be9f285b..c7410ca3d 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -152,12 +152,30 @@ requests waiting on it were never failed. -eight places that handed a message to a port did not take it back when the port -refused it. The port layer accepts a buffer only when it answers success, so on -a refusal -- which happens when memory runs out, or when the receiving process -has stopped reading a full shared queue -- the memory was lost. Two of them -lost shared memory that no cleanup reclaims, so an application could lose the -capacity to answer at all. +a message that could not be sent because the receiving process had died was +reported to the sender as sent. A sender that answers a request and then sends +a fallback lost both, so an application could wait for an answer that nothing +would ever deliver. + + + + + +eight places whose memory nothing else reclaims handed a message to a port and +did not take it back when the port refused it. The port layer accepts a buffer +only when it answers success, so on a refusal -- which happens when memory runs +out, or when the receiving process has stopped reading a full shared queue -- +the memory was lost. Two of them lost shared memory that no cleanup reclaims, +so an application could lose the capacity to answer at all. + + + + + +a websocket connection is now closed when a frame cannot be handed to the +application. Such a frame was previously dropped in silence, leaving the +peer's stream short a frame it was never told about -- which for a fragmented +message the client cannot detect. diff --git a/src/nxt_application.c b/src/nxt_application.c index 4f85da2dd..56c132fe1 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -1345,15 +1345,18 @@ nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process) * cleared only once the answer has actually been written or * queued, which is what makes the REMOVE_PID below streamless. * - * The failure that reaches this call site is an allocation - * failure, and it leaves no message behind. The prototype maps - * no queue for a router port -- nxt_app_new_port_handler() - * closes the queue descriptor every NEW_PORT carries, and main - * maps a queue only for an application port -- so - * nxt_port_socket_write2() never takes the shared-ring path that - * answers NXT_AGAIN, and what is left is - * nxt_port_msg_chk_insert() failing to allocate. Keeping - * ->stream then lets the REMOVE_PID carry it, and + * Three failures reach this call site, and none of them leaves + * a message behind. The prototype maps no queue for a router + * port -- nxt_app_new_port_handler() closes the queue descriptor + * every NEW_PORT carries, and main maps a queue only for an + * application port -- so nxt_port_socket_write2() never takes + * the shared-ring path that answers NXT_AGAIN. What is left is + * nxt_port_msg_chk_insert() failing to allocate; an inline + * write that hit EAGAIN with no memory left to hold it for a + * later attempt; and an inline write to a router port whose + * peer has died. + * + * Keeping ->stream then lets the REMOVE_PID carry it, and * nxt_router_remove_pid_handler() turns that into the same RPC * error -- the fallback the CREATED path has always used. * Clearing it regardless would leave the start RPC armed and @@ -1363,19 +1366,11 @@ nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process) * REMOVE_PID allocates a buffer and a message of its own from * the same pools and can fail the same way, and then only the * application's "limits.start_timeout" retires the start. The - * alert below is what makes that case visible instead of silent. - * One more shape reaches the same branch. The router port is - * writable, so the RPC_ERROR goes out inline and hits EAGAIN - * with no memory left to hold it for a later attempt. - * nxt_port_socket_write2() used to answer NXT_OK for that and - * drop the message, clearing the stream for a report nobody - * received. REMOVE_PID could not rescue it either: the same - * EAGAIN had cleared write_ready, so REMOVE_PID was queued - * instead of sent, and the error handler that ran next drained - * it too. It now answers NXT_ERROR with the message untouched, - * which keeps the stream armed here. A send to a port whose - * peer has died still answers NXT_OK; the RPC is moot there - * anyway. + * alert below is what makes that case visible instead of + * silent. After an EAGAIN, REMOVE_PID cannot rescue it either: + * the same EAGAIN cleared write_ready, so REMOVE_PID is queued + * rather than sent, and the error handler that runs next drains + * it too. */ if (nxt_fast_path(ret == NXT_OK)) { process->stream = 0; diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 5c9d09046..91f29528b 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -21,7 +21,7 @@ static nxt_int_t nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, nxt_port_send_msg_t *msg); static nxt_port_send_msg_t *nxt_port_msg_alloc(const nxt_port_send_msg_t *m); static nxt_int_t nxt_port_write_msgs(nxt_task_t *task, void *obj, - void *data); + void *data, nxt_bool_t *send_failed); static void nxt_port_write_handler(nxt_task_t *task, void *obj, void *data); static nxt_port_send_msg_t *nxt_port_msg_first(nxt_port_t *port); nxt_inline void nxt_port_msg_close_fd(nxt_port_send_msg_t *msg); @@ -220,10 +220,13 @@ nxt_port_release_send_msg(nxt_port_send_msg_t *msg) * nxt_port_error_handler() the caller raises next completes only what the * queue holds, and this message was never in it. * - * nxt_port_socket_write2() answers NXT_OK on these paths on purpose. - * Callers that ignore the return rely on the port to finish what they - * handed it, such as the START_PROCESS deadline in src/nxt_router.c. - * Skipping this call would orphan those buffers instead. + * nxt_port_socket_write2() answers NXT_OK on the paths that still reach + * here, on purpose. Callers that ignore the return rely on the port to + * finish what they handed it, such as the START_PROCESS deadline in + * src/nxt_router.c; skipping this call would orphan those buffers instead. + * The inline first fragment of a send to a dead peer no longer comes this + * way: nothing of it was consumed, so it is reported and stays the + * caller's. * * Close descriptors first, then complete buffers -- the order * nxt_port_error_handler() uses. @@ -264,7 +267,7 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, int notify; uint8_t qmsg_size; nxt_int_t res; - nxt_bool_t enqueued; + nxt_bool_t enqueued, send_failed; nxt_port_send_msg_t msg; struct { nxt_port_msg_t pm; @@ -272,6 +275,7 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, } qmsg; enqueued = 0; + send_failed = 0; msg.link.next = NULL; msg.link.prev = NULL; @@ -358,7 +362,7 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, * contract in src/nxt_port.h, and the same state this function * already answers for the allocation failure above. */ - res = nxt_port_write_msgs(task, &port->socket, &msg); + res = nxt_port_write_msgs(task, &port->socket, &msg, &send_failed); } if (nxt_slow_path(res != NXT_OK && enqueued)) { @@ -387,11 +391,23 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, * So record that a marker is owed and re-arm. nxt_port_announce() * sends it from the stack with no allocation, either here on this * port's own engine or from the item nxt_port_rearm() posts. + * + * Unless the send failed outright. Then it did not hit a transient + * condition: nxt_socketpair_send() answers NXT_AGAIN for EAGAIN and + * ENOBUFS and retries EINTR, so everything that reaches here is an + * error that will not get better -- a dead peer routinely, and + * EMSGSIZE, EBADF or a kernel ENOMEM otherwise. Every one of them + * ends this port: the marker can never be delivered, and re-arming + * the write event only brings this function back to fail the same + * send, leaving another queued error handler each pass while the + * event stays armed on a port that is about to be freed. */ - nxt_atomic_fetch_add(&port->announce, 1); + if (nxt_fast_path(!send_failed)) { + nxt_atomic_fetch_add(&port->announce, 1); - nxt_port_rearm(task, port); + nxt_port_rearm(task, port); + } res = NXT_OK; } @@ -676,20 +692,34 @@ nxt_port_rearm(nxt_task_t *task, nxt_port_t *port) * it was consumed. See the ownership contract this promises the caller in * the comment above the inline call in nxt_port_socket_write2(). * - * One exit answers NXT_ERROR today: an EAGAIN with no memory to hold a - * first fragment for a later attempt. A send to a dead peer still answers - * NXT_OK and drops the message; see the comment on that path below. + * Two exits answer NXT_ERROR, both for a first fragment sent inline: an + * EAGAIN with no memory to hold it for a later attempt, and a send that + * fails outright, which routinely means the peer is gone. + * + * "send_failed" separates them for the one caller that has to tell them + * apart, and is set on that second exit alone. It says what it tests: the + * sendmsg() failed with something nxt_socketpair_send() does not retry, of + * which a dead peer is the routine case. Such a port arms nothing -- it + * owes no queue marker, and re-arming its write event only schedules the + * same failing send again. + * + * The other three exits that raise the error handler -- a short write, and + * two allocation failures after a fragment has gone out -- leave a socket + * that still works, so they keep re-arming on an owed marker. The handler + * itself does not decide this either way: it drains the port's queued + * messages and drops one reference, and closes nothing. */ static nxt_int_t -nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data) +nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, + nxt_bool_t *send_failed) { int use_delta; size_t plain_size; ssize_t n; nxt_int_t ret; uint32_t mmsg_buf[3 * NXT_IOBUF_MAX * 10]; - nxt_bool_t block_write, enable_write; + nxt_bool_t failed, block_write, enable_write; nxt_port_t *port; struct iovec iov[NXT_IOBUF_MAX * 10]; nxt_work_queue_t *wq; @@ -700,6 +730,7 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data) port = nxt_container_of(obj, nxt_port_t, socket); ret = NXT_OK; + failed = 0; block_write = 0; enable_write = 0; use_delta = 0; @@ -839,21 +870,31 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data) } else { /* - * A send that failed outright: the peer is gone. This keeps the - * old treatment -- drop the message here and answer NXT_OK -- - * even though the caller is then told a message went out that - * did not. It is the routine path, taken by every worker that - * dies with a send in flight, and the callers that ignore this - * return were written for it: they hand over a buffer and rely - * on the port to complete it. Reporting it instead would leak - * those buffers, one of them a chain of shared-memory chunks - * that no pool teardown reclaims. It has to be done together - * with those call sites; #348 tracks that. + * A send that failed outright -- routinely a dead peer, but + * nxt_socketpair_send() retries or defers everything except a + * non-recoverable errno, so this covers all of them. Nothing of + * the message was consumed -- nxt_socketpair_send() sent no + * bytes and no descriptor, and nxt_port_msg_close_fd() runs only + * after a successful send -- so an inline first fragment is + * still whole and still the caller's. Saying so is what lets + * the caller stop waiting for a reply that is not coming. + * + * Once a fragment has gone out the message is no longer whole, + * and a queued message has no caller to tell, so both keep the + * old treatment: the port owns them, and nxt_port_msg_drop() + * finishes them. */ if (nxt_slow_path(n == NXT_ERROR)) { + failed = 1; + if (msg->link.next == NULL) { - nxt_port_msg_drop(task, msg); + if (data != NULL && msg->port_msg.nf == 0) { + ret = NXT_ERROR; + + } else { + nxt_port_msg_drop(task, msg); + } } goto fail; @@ -934,7 +975,14 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data) * once more. */ - if (enable_write || port->announce != 0) { + /* + * Not on a port whose send failed outright, and that covers the + * event-loop pass as well as the inline one: both reach this through + * the same "fail:" label, and neither has anything to gain from arming + * a socket that has just refused to carry a message. + */ + + if (!failed && (enable_write || port->announce != 0)) { nxt_port_rearm(task, port); } @@ -942,6 +990,10 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data) nxt_port_use(task, port, use_delta); } + if (send_failed != NULL) { + *send_failed = failed; + } + return ret; } @@ -949,7 +1001,7 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data) static void nxt_port_write_handler(nxt_task_t *task, void *obj, void *data) { - (void) nxt_port_write_msgs(task, obj, data); + (void) nxt_port_write_msgs(task, obj, data, NULL); } diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index 35d4c8325..f416f3ab7 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -19,6 +19,7 @@ static nxt_int_t nxt_port_fail_test_socket_write(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_inline_drop(nxt_thread_t *thr); static void nxt_port_fail_test_enable_write(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static nxt_int_t nxt_port_fail_test_dead_peer(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_error_handler(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_mp_baseline(nxt_thread_t *thr); @@ -65,6 +66,10 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_dead_peer(thr) != NXT_OK) { + return NXT_ERROR; + } + if (nxt_port_fail_test_rpc_register(thr) != NXT_OK) { return NXT_ERROR; } @@ -625,6 +630,240 @@ nxt_port_fail_test_inline_drop(nxt_thread_t *thr) } +/* + * The same wake-up, to a peer that is gone. + * + * The payload reaches the shared queue exactly as above, so it is the port's + * whatever becomes of the wake-up and the answer is still NXT_OK. What + * differs is everything the port does afterwards. An EAGAIN leaves a socket + * that will take writes again, so the marker is owed and the write event is + * re-armed. A send that fails outright leaves a peer that will never read + * the ring, and nxt_port_write_msgs() has raised the error handler on it. + * + * Owing a marker there is not merely useless. The re-arm enables the write + * event, the next pass finds announce non-zero and sends the marker again, + * and each pass is another failed send on a port the error handler is tearing + * down -- until the event fires on a port that has been freed. That is what + * the application churn tests saw: repeated EPIPE, then pthread_mutex_lock() + * failing with EINVAL on the write mutex of a destroyed port. + */ + +static nxt_int_t +nxt_port_fail_test_dead_peer(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_fd_t pair[2]; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine, *saved_engine; + nxt_port_queue_t *queue; + nxt_event_interface_t stub; + u_char qbuf[NXT_PORT_QUEUE_MSG_SIZE]; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + pair[0] = -1; + pair[1] = -1; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_fail_test_enable_write; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + port->engine = &engine; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: socketpair failed"); + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.fd = pair[1]; + port->socket.task = task; + port->socket.log = thr->log; + port->max_size = 1024; + port->max_share = 1024; + + queue = nxt_mp_zalloc(mp, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue == NULL)) { + goto done; + } + + nxt_port_queue_init(queue); + port->queue = queue; + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_fail_test_completion; + buf->mem.free++; + + nxt_port_fail_test_completions = 0; + nxt_port_fail_test_rearms = 0; + + /* + * The reader end goes now, so the wake-up fails outright rather than + * with EAGAIN. Both copies of the number are cleared with it: the + * teardown below closes whatever port->pair[0] still holds, and closing + * a number twice would hand this fixture somebody else's descriptor. + */ + + nxt_fd_close(pair[0]); + pair[0] = -1; + port->pair[0] = -1; + + port->socket.write_ready = 1; + port->socket.write = NXT_EVENT_INACTIVE; + + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 0, 0, buf); + + if (ret != NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: a payload that reached the shared " + "queue answered %d after the peer was gone; the buffer " + "is already the port's and the caller must not be told " + "to reclaim it", (int) ret); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_ERROR; + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_fail_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the enqueued payload was completed " + "%d times, expected once", + (int) nxt_port_fail_test_completions); + goto done; + } + + if (port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: a queue marker was owed to a peer " + "that is gone (announce %d); every later pass resends " + "it on a port the error handler is tearing down", + (int) port->announce); + goto done; + } + + if (nxt_port_fail_test_rearms != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the write event was re-armed %d " + "times on a failed port, expected none", + (int) nxt_port_fail_test_rearms); + goto done; + } + + /* + * The same port, now owing a marker from before the peer died -- the + * state an earlier EAGAIN leaves, and the one the re-arm exists to + * settle. A failed pass must not settle it: the debt is read again on + * the way out of every write, so a port that keeps re-arming on it sends + * one more doomed marker per pass. + * + * The ring is drained first only so the next payload is again the item + * that raises "notify"; nxt_port_queue_send() raises it on the 0-to-1 + * transition alone. + */ + + while (nxt_port_queue_recv(queue, qbuf) >= 0) { + /* void */ + } + + nxt_atomic_fetch_add(&port->announce, 1); + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_fail_test_completion; + buf->mem.free++; + + nxt_port_fail_test_completions = 0; + + port->socket.write_ready = 1; + + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 0, 0, buf); + + if (ret != NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the second enqueued payload " + "answered %d", (int) ret); + ret = NXT_ERROR; + goto done; + } + + ret = NXT_ERROR; + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_fail_test_rearms != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: a failed port re-armed to send a " + "marker it already owed (%d times)", + (int) nxt_port_fail_test_rearms); + goto done; + } + + if (port->announce != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the owed marker was counted again " + "on a port that is going away (announce %d)", + (int) port->announce); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + port->queue = NULL; + + nxt_port_close(task, port); + + nxt_port_use(task, port, -1); + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_mp_destroy(mp); + + return ret; +} + + static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr) { @@ -1702,6 +1941,7 @@ nxt_port_fail_test_cross_engine_rearm(nxt_thread_t *thr) return ret; } + /* * A caller gets its buffer back when the port refuses the message. * @@ -1815,6 +2055,7 @@ nxt_port_fail_test_caller_owns_buf(nxt_thread_t *thr) return ret; } + static nxt_int_t nxt_port_fail_test_queued(nxt_locked_work_queue_t *lwq, nxt_work_t *item) { diff --git a/src/test/nxt_router_start_timeout_test.c b/src/test/nxt_router_start_timeout_test.c index 289792d3b..f3d2cb84b 100644 --- a/src/test/nxt_router_start_timeout_test.c +++ b/src/test/nxt_router_start_timeout_test.c @@ -843,23 +843,22 @@ nxt_router_start_timeout_test(nxt_thread_t *thr) } /* - * A start whose write is accepted and then dropped inside the port layer. + * A start whose write the port layer refuses. * * With the port write-ready and its queue empty, nxt_port_msg_chk_insert() - * declines to queue and nxt_port_socket_write2() sends inline through - * nxt_port_write_handler(). A send that fails there -- here because the - * descriptor is -1, in the field because the peer is gone, or because - * nxt_port_msg_insert_tail() cannot allocate after an NXT_AGAIN, which - * happens against a perfectly live port -- drops a message that was never - * in port->messages. write2() still answers NXT_OK, because the handler - * returns void. + * declines to queue and nxt_port_socket_write2() sends inline. A send that + * fails there -- here because the descriptor is -1, in the field because + * the peer is gone -- leaves a message that was never in port->messages and + * was never consumed, so write2() answers NXT_ERROR. * - * So the deadline finds nothing to cancel and, waiting for the payload to - * be released, would wait for a completion nobody was going to run: the - * RPC and its pending_processes slot armed for good, which is the very - * wedge the option exists to bound. nxt_port_msg_drop() completes the - * buffers of these dropped messages, so the wait ends the way every other - * one does. + * This start therefore needs no deadline: the caller takes its own + * "failed:" path, frees the payload it still owns, and gives the + * pending_processes slot back inside the call. + * + * The assertion that matters is the negative one below. The port layer + * must not have completed that payload on its way out, because the caller + * frees it with nxt_mp_free() -- a completion queued here would run + * against freed memory. */ app->pending_processes = 1; @@ -889,28 +888,50 @@ nxt_router_start_timeout_test(nxt_thread_t *thr) goto done; } - /* The completion the drop queued, and the error handler it raised. */ - - (void) nxt_router_start_timeout_test_drain(task, &engine); + /* + * One work item: the error handler the dead socket raised. Not two -- + * a buffer completion queued here would run against memory the caller + * has already handed to nxt_mp_free(), which is the double free the + * ownership contract exists to prevent, and it is why this leg counts + * rather than merely draining. + */ - ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + ran = nxt_router_start_timeout_test_drain(task, &engine); - if (ran == 0) { + if (ran != 1) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "router start timeout test: nothing bounded a start " - "whose write was dropped inside the port layer"); + "router start timeout test: a refused write queued %d " + "work items, expected 1 -- the port layer completed a " + "payload the caller still owns and has freed", + (int) ran); goto done; } if (app->proto_port_requests != 0 || app->pending_processes != 0) { nxt_log_error(NXT_LOG_NOTICE, thr->log, "router start timeout test: a start whose write was " - "dropped stayed armed, proto_port_requests %uD pending " + "refused stayed armed, proto_port_requests %uD pending " "%uD (expected 0 and 0)", app->proto_port_requests, app->pending_processes); goto done; } + /* + * And no deadline was needed to get there: the refusal failed the start + * inside the call. A timer firing here would mean it had been left armed + * for one to find, which is the state this leg exists to rule out. + */ + + ran = nxt_router_start_timeout_test_tick(task, &engine, 2000); + + if (ran != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start timeout test: a deadline fired for a start " + "the port layer had already refused (%d handlers)", + (int) ran); + goto done; + } + /* * And the same expiry on the config-apply path, which is where the payload * has a lifetime of its own: nxt_router_app_rpc_create() allocates it from From 4d3c365a0121df0248b8c6b703cb3788f9b84e71 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 16:21:08 +0200 Subject: [PATCH 194/391] build(deps): bump contrib njs to 1.0.1 njs 1.0.1 (2 Sep 2026) fixes three CVEs: an access-control bypass in js_access on an async body exception (CVE-2026-18329), a worker crash reading Response.statusText with an empty reason phrase (CVE-2026-78222), and a heap buffer overflow in xml.exclusiveC14n() (CVE-2026-78689). Most of the remaining fixes are QuickJS-engine specific and do not affect Unit's --no-quickjs build. Blocked on two prerequisites, not fixed by this commit: - freeunitorg/njs has no 1.0.1 tag yet, so .github/workflows/build-test.yml cannot check out njs for this version until the tag is pushed. - packages.freeunit.org has no njs-1.0.1.tar.gz yet, so the contrib Makefile's own download cannot succeed until it is published there. SHA512SUMS carries no digest for 1.0.1 yet. A digest computed from the upstream GitHub tag archive cannot substitute: checked against 1.0.0, the mirror tarball is 1,007,848 bytes and the GitHub tag archive for the same tag is 1,011,616 bytes, with different SHA512 digests, even though the mirror's own recorded 1.0.0 digest matches its own tarball exactly. The 1.0.1 digest must be computed from the real njs-1.0.1.tar.gz once it reaches the mirror. Verified: built contrib njs 1.0.1 locally from the upstream GitHub tag tarball with the existing --no-quickjs --no-libxml2 --no-zlib flags, built unitd --njs against it, and ran test/test_njs.py and test/test_njs_modules.py (12/12 passed). All njs_* symbols used in src/nxt_js.c, src/nxt_http_js.c and src/nxt_script.c are still present in njs 1.0.1's public headers (grep -hoE 'njs_[a-zA-Z0-9_]+' over those three files | sort -u: 40 distinct tokens, a broad match including types like njs_vm_t and njs_int_t, not just callable API functions). auto/njs's NJS_VERSION_NUMBER >= 0x000900 guard is satisfied by 1.0.1's 0x010001. Related to #121. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 2 ++ docs/changes.xml | 6 ++++++ pkg/contrib/src/njs/Makefile | 2 +- pkg/contrib/src/njs/SHA512SUMS | 9 ++++++++- pkg/contrib/src/njs/version | 2 +- 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 76b4a1a1f..db57e9777 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Change: upgrade contrib njs to 1.0.1. + *) Bugfix: a static response that was subject to content negotiation now carries "Vary: Accept-Encoding". Without it a shared cache could store whichever representation it saw first and serve it to every client -- diff --git a/docs/changes.xml b/docs/changes.xml index c7410ca3d..f22d5c903 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,12 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +upgrade contrib njs to 1.0.1. + + + removed the unused job cluster (nxt_job.c, nxt_job.h, and diff --git a/pkg/contrib/src/njs/Makefile b/pkg/contrib/src/njs/Makefile index 431ab4ec9..e5bd83c36 100644 --- a/pkg/contrib/src/njs/Makefile +++ b/pkg/contrib/src/njs/Makefile @@ -14,7 +14,7 @@ njs: njs-$(NJS_VERSION).tar.gz .sum-njs $(UNPACK) $(MOVE) -# njs 1.0.0 recommends QuickJS, but that requires an external QuickJS / +# njs 1.0.1 recommends QuickJS, but that requires an external QuickJS / # QuickJS-NG library, so build --no-quickjs to keep the baseline deterministic. .njs: njs cd $< && ./configure \ diff --git a/pkg/contrib/src/njs/SHA512SUMS b/pkg/contrib/src/njs/SHA512SUMS index ab99c9d18..67fe10215 100644 --- a/pkg/contrib/src/njs/SHA512SUMS +++ b/pkg/contrib/src/njs/SHA512SUMS @@ -1 +1,8 @@ -fde328f79869821fd288a0a19afee44e4089959513aa79f5831f44f4be4a562312cd04d3d27467bae4b909341a962533f54920b71c533ed58122bdb3bf9b335d njs-1.0.0.tar.gz +# No digest recorded yet for njs-1.0.1.tar.gz. This file must record the +# SHA512 of the tarball served from packages.freeunit.org/njs (the URL the +# contrib Makefile actually downloads), not a GitHub tag archive: verified +# against 1.0.0, where the mirror tarball is 1,007,848 bytes and the GitHub +# tag archive for the same tag is 1,011,616 bytes -- a different file with a +# different digest, even though the mirror's own recorded digest below +# matches its tarball exactly. Compute this line's digest from the real +# njs-1.0.1.tar.gz once it is published to the mirror. diff --git a/pkg/contrib/src/njs/version b/pkg/contrib/src/njs/version index 5e8307059..fa2b1d135 100644 --- a/pkg/contrib/src/njs/version +++ b/pkg/contrib/src/njs/version @@ -1 +1 @@ -NJS_VERSION := 1.0.0 +NJS_VERSION := 1.0.1 From 9d0c2323ac15bee96a3768b92bbbd9d428e0f6d0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 18:06:36 +0200 Subject: [PATCH 195/391] build(deps): record the njs 1.0.1 digest The tarball is published now, so the placeholder comment becomes the line it described. The digest is the one the mirror serves, which is what the Makefile downloads. Four sources agree on it: "git archive --prefix=njs-1.0.1/ ac8ff3a6" run locally, the GitHub release asset the fork's workflow built from that commit, ./update_hash.sh in the packages repository, and the file fetched from packages.freeunit.org. Verified through the build rather than by hand: "make -C pkg/contrib njs" downloads the tarball and reports "njs-1.0.1.tar.gz: OK". Co-Authored-By: Claude Opus 5 (1M context) --- pkg/contrib/src/njs/SHA512SUMS | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pkg/contrib/src/njs/SHA512SUMS b/pkg/contrib/src/njs/SHA512SUMS index 67fe10215..04d2143ff 100644 --- a/pkg/contrib/src/njs/SHA512SUMS +++ b/pkg/contrib/src/njs/SHA512SUMS @@ -1,8 +1 @@ -# No digest recorded yet for njs-1.0.1.tar.gz. This file must record the -# SHA512 of the tarball served from packages.freeunit.org/njs (the URL the -# contrib Makefile actually downloads), not a GitHub tag archive: verified -# against 1.0.0, where the mirror tarball is 1,007,848 bytes and the GitHub -# tag archive for the same tag is 1,011,616 bytes -- a different file with a -# different digest, even though the mirror's own recorded digest below -# matches its tarball exactly. Compute this line's digest from the real -# njs-1.0.1.tar.gz once it is published to the mirror. +0339134b30a382820ff7678975425bf1f7040f3f52304b698f7cf08717d07cd9988508982bcd7a8e5e1d64fbe2ad634e38587d37f1afdbe9e9a8ffe8915ac667 njs-1.0.1.tar.gz From 0f83875a5d88d2a409a40e1a26ef376ff3038d50 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 06:57:09 +0200 Subject: [PATCH 196/391] http: escape what a variable expands to in a text access log The string form of "access_log" "format" resolved the whole template and wrote the result to the file. Nothing looked at the bytes on the way. "$uri" is r->path, the percent-decoded target, so the client chooses them: GET /%0d%0a1.2.3.4%20%22FORGED%22%20200 HTTP/1.1 Measured against this format, "$remote_addr \"$uri\" $status": before: 3 lines, the second reading 1.2.3.4 "FORGED" 200 after: 1 line, the CRLF shown as \x0D\x0A Escape the bytes a variable expands to, and leave the template around them alone. nxt_var_interpreter() is the only place that can tell the two apart: literal template bytes are copied from the compiled template, values from what the handler returned. It gets a flags word instead of the logging bool, so the escaping reaches only the format that asks for it. The object form is not touched. It prints through nxt_conf_json_print(), which escapes what it prints, so escaping again would show the escape instead of the value. Both forms compile with NXT_TSTR_LOGGING, so the new NXT_TSTR_ESCAPE is what separates them. Escape exactly what nginx escapes, and the way nginx writes it (ngx_http_log_escape): the quote, the backslash, every byte below 0x20, 0x7F and the whole high half, each as "\xHH". nginx has one emit path and no short forms, so a reader that already unescapes an nginx log reads this one; writing \" instead would make such a reader see a literal backslash followed by a field-terminating quote, which moves the field boundary an attacker-controlled value sits in. The high half is not tidiness either: a reader that decodes the log as UTF-8 and splits on Unicode line breaks treats U+0085, U+2028 and U+2029 as terminators. An njs format produces the whole record, template included, so all of it is escaped -- except the trailing newline, which nxt_js_add_tpl() appends inside the generated function. The flag has to be normalised to 0 or 1 before it is used as that length: nxt_bool_t is nxt_uint_t, so passing the masked bit reported the record three bytes longer than it was written and nxt_fd_write() sent three bytes of pool memory after every escaped njs record. Measured on an njs build, same request: 33 bytes written for a 30-byte record, the last three uninitialised. Both hand-written two-pass sites now assert that they wrote exactly the length they claimed. A value that carries a quote of its own now shows it escaped, which test_variables_response_header caught: an entity-tag is a quoted string, so "$response_header_etag" now logs \x22abc\x22 where it logged "abc". Tested on an njs build, all three paths: a string format leaves one record with the CRLF escaped, an njs format leaves one record of exactly its own length -- asserted on the raw bytes, since three stray bytes that happened to be newlines would vanish into a line split -- and an object format still emits JSON that parses, carrying "\r\n" rather than a second escaping. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 21 ++++++++ docs/changes.xml | 21 ++++++++ src/nxt_router_access_log.c | 10 +++- src/nxt_tstr.c | 83 ++++++++++++++++++++++++++++++-- src/nxt_tstr.h | 7 +++ src/nxt_var.c | 96 ++++++++++++++++++++++++++++++++++--- src/nxt_var.h | 13 ++++- test/test_access_log.py | 78 ++++++++++++++++++++++++++++++ test/test_variables.py | 6 ++- 9 files changed, 323 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index db57e9777..175fd9a7f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,27 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Security: the string form of "access_log" "format" wrote what a + variable expanded to without escaping it. "$uri" is the + percent-decoded target, so a request for "/%0d%0a..." put a real + CRLF in the log and the rest of the line read as a record of a + request that never happened. Bytes a variable expands to are now + escaped the way nginx escapes them: the quote, the backslash, every + byte below 0x20, 0x7F and every byte above it all become "\xHH". + nginx emits no short forms, so a reader that already unescapes an + nginx log reads this one. The text of the format itself is + unchanged, so a tab or a quote an operator wrote still reaches the + log as itself; an "njs" format is the exception, because it + produces the whole record and all of it is escaped. A value that + carries a quote of its own now shows it escaped: an entity-tag + logged through "$response_header_etag" reads \x22abc\x22 rather + than "abc", and a non-ASCII path reads "\xD0\xBF..." rather than + its bytes. A log parser that expected the raw bytes has to be + updated, though "\xHH" is reversible and the bytes can be recovered + from it. The object form of "format" is unaffected by this change, + but it is not a way to keep bytes either: it replaces anything that + is not valid UTF-8 with U+FFFD before it serializes. + *) Change: upgrade contrib njs to 1.0.1. *) Bugfix: a static response that was subject to content negotiation now diff --git a/docs/changes.xml b/docs/changes.xml index f22d5c903..7bd172beb 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,27 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +the string form of "access_log" "format" wrote what a variable expanded to +without escaping it. "$uri" is the percent-decoded target, so a request for +"/%0d%0a..." put a real CRLF in the log and the rest of the line read as a +record of a request that never happened. Bytes a variable expands to are now +escaped the way nginx escapes them: the quote, the backslash, every byte below +0x20, 0x7F and every byte above it all become "\xHH". nginx emits no short +forms, so a reader that already unescapes an nginx log reads this one. The +text of the +format itself is unchanged, except for an "njs" format, which produces the +whole record and is escaped whole. A value that carries a quote of its own now +shows it escaped: an entity-tag logged through "$response_header_etag" reads +\x22abc\x22 rather than "abc", and a non-ASCII path reads "\xD0\xBF..." rather than +its bytes. A log parser that expected the raw bytes has to be updated, though "\xHH" is +reversible and the bytes can be recovered from it. The object form of "format" +is unaffected by this change, but it is not a way to keep bytes either: it +replaces anything that is not valid UTF-8 with U+FFFD before it serializes. + + + upgrade contrib njs to 1.0.1. diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index b12e55f54..45ae8f61a 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -220,8 +220,16 @@ nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, str = default_format; } + /* + * NXT_TSTR_ESCAPE is set here and not for the object format above: the + * object format prints through nxt_conf_json_print(), which escapes what + * it prints, and escaping twice would show the escape instead of the + * value. + */ + format->tstr = nxt_tstr_compile(rtcf->tstr_state, &str, - NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE); + NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE + | NXT_TSTR_ESCAPE); if (nxt_slow_path(format->tstr == NULL)) { return NULL; } diff --git a/src/nxt_tstr.c b/src/nxt_tstr.c index d939d44e2..b99ef1de6 100644 --- a/src/nxt_tstr.c +++ b/src/nxt_tstr.c @@ -261,11 +261,66 @@ nxt_tstr_query_init(nxt_tstr_query_t **query_p, nxt_tstr_state_t *state, } +#if (NXT_HAVE_NJS) + +/* + * A JS template produces the whole record, so there is no template text to + * keep apart from the values -- escape all of it. The trailing newline is + * part of that value: nxt_js_add_tpl() appends it inside the generated + * function, so escaping it too would turn every record into one line. + */ + +static nxt_int_t +nxt_tstr_escape(nxt_mp_t *mp, nxt_str_t *val, nxt_bool_t newline) +{ + size_t length, size; + u_char *p; + nxt_str_t escaped; + + length = val->length; + + if (newline && length > 0 && val->start[length - 1] == '\n') { + length--; + + } else { + newline = 0; + } + + size = nxt_var_escape_length(val->start, length); + + if (size == length) { + return NXT_OK; + } + + escaped.length = size + newline; + + escaped.start = nxt_mp_nget(mp, escaped.length); + if (nxt_slow_path(escaped.start == NULL)) { + return NXT_ERROR; + } + + p = nxt_var_escape(escaped.start, val->start, length); + + if (newline) { + *p++ = '\n'; + } + + nxt_assert(p == escaped.start + escaped.length); + + *val = escaped; + + return NXT_OK; +} + +#endif + + nxt_int_t nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, nxt_str_t *val) { - nxt_int_t ret; + nxt_int_t ret; + nxt_uint_t flags; if (nxt_tstr_is_const(tstr)) { nxt_tstr_str(tstr, val); @@ -273,9 +328,18 @@ nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, } if (tstr->type == NXT_TSTR_VAR) { + flags = 0; + + if (tstr->flags & NXT_TSTR_LOGGING) { + flags |= NXT_VAR_LOGGING; + } + + if (tstr->flags & NXT_TSTR_ESCAPE) { + flags |= NXT_VAR_ESCAPE; + } + ret = nxt_var_interpreter(task, query->state, &query->cache->var, - tstr->u.var, val, query->ctx, - tstr->flags & NXT_TSTR_LOGGING); + tstr->u.var, val, query->ctx, flags); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; @@ -289,6 +353,19 @@ nxt_tstr_query(nxt_task_t *task, nxt_tstr_query_t *query, nxt_tstr_t *tstr, if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; } + + if (tstr->flags & NXT_TSTR_ESCAPE) { + /* + * (flags & BIT) != 0, not the masked bit: nxt_bool_t is + * nxt_uint_t, and nxt_tstr_escape() adds this value to a length. + * nxt_tstr_compile() normalises the same flag the same way. + */ + ret = nxt_tstr_escape(query->cache->var.pool, val, + (tstr->flags & NXT_TSTR_NEWLINE) != 0); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + } #endif } diff --git a/src/nxt_tstr.h b/src/nxt_tstr.h index 7c067cdba..b96d04a31 100644 --- a/src/nxt_tstr.h +++ b/src/nxt_tstr.h @@ -35,6 +35,13 @@ typedef enum { NXT_TSTR_STRZ = 1 << 0, NXT_TSTR_LOGGING = 1 << 1, NXT_TSTR_NEWLINE = 1 << 2, + /* + * Escape what a variable expands to, leaving the template around it as + * the operator wrote it. The access log's string format sets this: its + * values come from the client, and a raw newline in one of them ends the + * record and starts a line the client chose. + */ + NXT_TSTR_ESCAPE = 1 << 3, } nxt_tstr_flags_t; diff --git a/src/nxt_var.c b/src/nxt_var.c index cb422380f..bdb7f17f0 100644 --- a/src/nxt_var.c +++ b/src/nxt_var.c @@ -493,10 +493,75 @@ nxt_var_next_part(u_char *start, u_char *end, nxt_str_t *part) } +/* + * Escape one byte of a variable's value the way nginx's access log does + * (ngx_http_log_escape()): the quote, the backslash and everything outside + * printable ASCII become "\xHH". nginx has one emit path and no short forms + * such as "\\\"", so a reader that already unescapes an nginx log reads this + * one correctly; emitting "\\\"" here would make such a reader see a literal + * backslash followed by a field-terminating quote. + * + * The high half is escaped too, not only the control characters. A reader + * that decodes the log as UTF-8 and splits it on Unicode line breaks treats + * U+0085, U+2028 and U+2029 as terminators, so a raw 0xC2 0x85 in a value + * would end a record just as a raw newline does. + */ + +nxt_inline nxt_bool_t +nxt_var_escaped(u_char c) +{ + return c < 0x20 || c >= 0x7F || c == '"' || c == '\\'; +} + + +size_t +nxt_var_escape_length(const u_char *p, size_t length) +{ + size_t size, i; + + size = length; + + for (i = 0; i < length; i++) { + if (nxt_var_escaped(p[i])) { + /* One byte becomes the four of "\xHH". */ + size += 3; + } + } + + return size; +} + + +u_char * +nxt_var_escape(u_char *dst, const u_char *src, size_t length) +{ + static const u_char hex[] = "0123456789ABCDEF"; + + size_t i; + u_char c; + + for (i = 0; i < length; i++) { + c = src[i]; + + if (!nxt_var_escaped(c)) { + *dst++ = c; + continue; + } + + *dst++ = '\\'; + *dst++ = 'x'; + *dst++ = hex[c >> 4]; + *dst++ = hex[c & 0xf]; + } + + return dst; +} + + nxt_int_t nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, nxt_var_cache_t *cache, nxt_var_t *var, nxt_str_t *str, void *ctx, - nxt_bool_t logging) + nxt_uint_t flags) { u_char *p, *src; size_t length, last, next; @@ -529,9 +594,16 @@ nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, *part = value; - length += value->length - subs[i].length; + length -= subs[i].length; - if (logging && value->start == NULL) { + if (flags & NXT_VAR_ESCAPE) { + length += nxt_var_escape_length(value->start, value->length); + + } else { + length += value->length; + } + + if ((flags & NXT_VAR_LOGGING) && value->start == NULL) { length += 1; } } @@ -556,9 +628,14 @@ nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, p = nxt_cpymem(p, &src[last], next - last); } - p = nxt_cpymem(p, part[i]->start, part[i]->length); + if (flags & NXT_VAR_ESCAPE) { + p = nxt_var_escape(p, part[i]->start, part[i]->length); - if (logging && part[i]->start == NULL) { + } else { + p = nxt_cpymem(p, part[i]->start, part[i]->length); + } + + if ((flags & NXT_VAR_LOGGING) && part[i]->start == NULL) { *p++ = '-'; } @@ -566,9 +643,16 @@ nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, } if (last != var->length) { - nxt_cpymem(p, &src[last], var->length - last); + p = nxt_cpymem(p, &src[last], var->length - last); } + /* + * The pass above and the length computed for nxt_mp_nget() are written by + * hand and have to agree. Say so, rather than letting them drift apart + * quietly: what a variable expands to is escaped in one of them. + */ + nxt_assert(p == str->start + str->length); + return NXT_OK; } diff --git a/src/nxt_var.h b/src/nxt_var.h index ca86513cb..013654420 100644 --- a/src/nxt_var.h +++ b/src/nxt_var.h @@ -58,9 +58,20 @@ nxt_var_field_t *nxt_var_field_new(nxt_mp_t *mp, nxt_str_t *name, nxt_var_t *nxt_var_compile(nxt_tstr_state_t *state, nxt_str_t *str); nxt_int_t nxt_var_test(nxt_tstr_state_t *state, nxt_str_t *str, u_char *error); +typedef enum { + /* Write "-" for a variable that has no value, as an access log does. */ + NXT_VAR_LOGGING = 1 << 0, + /* Escape the bytes a variable expands to; see nxt_var_escape(). */ + NXT_VAR_ESCAPE = 1 << 1, +} nxt_var_flags_t; + + +size_t nxt_var_escape_length(const u_char *p, size_t length); +u_char *nxt_var_escape(u_char *dst, const u_char *src, size_t length); + nxt_int_t nxt_var_interpreter(nxt_task_t *task, nxt_tstr_state_t *state, nxt_var_cache_t *cache, nxt_var_t *var, nxt_str_t *str, void *ctx, - nxt_bool_t logging); + nxt_uint_t flags); nxt_str_t *nxt_var_get(nxt_task_t *task, nxt_tstr_state_t *state, nxt_var_cache_t *cache, nxt_str_t *name, void *ctx); diff --git a/test/test_access_log.py b/test/test_access_log.py index ac2ce569b..6932bd57f 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -315,6 +315,67 @@ def _read_log_lines(): return [ln for ln in f.read().splitlines() if ln] +def test_access_log_text_escape(wait_for_record): + load('empty') + + # A percent-encoded CRLF in the target reaches "$uri" decoded, so without + # escaping these bytes end the record and start one the client wrote. + set_format('$remote_addr "$uri" $status') + + assert ( + client.get(url='/%0d%0a1.2.3.4 "GET /forged" 200')['status'] == 200 + ) + assert wait_for_record(r'\\x0D\\x0A', 'access.log') is not None + + lines = _read_log_lines() + + assert len(lines) == 1, 'one request, one record' + assert '\\x0D\\x0A' in lines[0], 'the CRLF is escaped, not written' + assert '\\x22GET /forged\\x22 200' in lines[0], 'the quotes are escaped too' + + +def test_access_log_text_escape_leaves_the_format_alone(wait_for_record): + load('empty') + + # The operator's own format text is not a value and is written as it is; + # only what a variable expands to is escaped. + set_format('start\t"$uri"\tend') + + assert client.get(url='/ok')['status'] == 200 + assert wait_for_record(r'start', 'access.log') is not None + + line = _read_log_lines()[-1] + + assert line == 'start\t"/ok"\tend', 'tabs in the format survive' + + +def test_access_log_text_escape_njs(require, wait_for_record): + require({'modules': {'njs': 'any'}}) + + load('empty') + + # An njs format produces the whole record, so all of it is escaped -- + # the quotes written here included. The client's CRLF must not reach the + # file, and the record must be exactly as long as it claims: the newline + # it ends with is appended inside the generated function, so a length that + # counted the flag bit instead of a 0/1 would report pool memory past the + # end of the record. + set_format('`${vars.remote_addr} "${vars.uri}"`') + + assert client.get(url='/%0d%0aFORGED')['status'] == 200 + assert wait_for_record(r'FORGED', 'access.log') is not None + + # On the raw bytes, not on split lines: three stray bytes that happened to + # be newlines would vanish into a line split and the record would still + # look right. + with open(f'{option.temp_dir}/access.log', 'rb') as f: + raw = f.read() + + assert ( + raw == b'127.0.0.1 \\x22/\\x0D\\x0AFORGED\\x22\n' + ), 'one record, one newline, nothing after' + + def test_access_log_object_escape(wait_for_record): load('empty') @@ -347,6 +408,23 @@ def test_access_log_object_escape(wait_for_record): assert record['ua'] == evil, 'value round-trips' +def test_access_log_object_escape_is_not_the_text_escape(wait_for_record): + load('empty') + + # The other half of the split: the object format escapes as JSON and must + # not also take the text format's escaping, or a reader would get "\x0D" + # where JSON says "\r" and the member would no longer round-trip. + set_format({'uri': '$uri', 'status': '$status'}) + + assert client.get(url='/%0d%0aFORGED')['status'] == 200 + assert wait_for_record(r'FORGED', 'access.log') is not None + + record = json.loads(_read_log_lines()[-1]) + + assert record['uri'] == '/\r\nFORGED', 'the bytes round-trip through JSON' + assert '\\x0D' not in record['uri'], 'not escaped twice' + + def test_access_log_object_escape_njs(require, wait_for_record): require({'modules': {'njs': 'any'}}) diff --git a/test/test_variables.py b/test/test_variables.py index 379357a1a..31cff538a 100644 --- a/test/test_variables.py +++ b/test/test_variables.py @@ -577,10 +577,14 @@ def test_variables_response_header(temp_dir, wait_for_record): # response clock, not a file's, so it stays a shape. last_modified = re.escape(formatdate(mtime, usegmt=True)) + # An entity-tag is a quoted string, and a text access log escapes the + # quote inside a value as \x22, the way nginx escapes every byte it + # escapes. That is what keeps a quoted field parseable when the value + # carries a quote of its own. assert client.get(url='/foo/index.html')['status'] == 200 assert ( wait_for_record( - rf'share@{last_modified}@".*"@text/html@Unit/.*@.*GMT@5@close', + rf'share@{last_modified}@\\x22.*\\x22@text/html@Unit/.*@.*GMT@5@close', 'access.log', ) is not None From 2ee5ee6923ff026b95926ae30e6e9d93e4e1bb04 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 16:32:58 +0200 Subject: [PATCH 197/391] test: default "Connection: close" for explicit request headers The default header dict carries "Connection: close"; passing an explicit dict replaced it, and without the header the one-shot client reads to EOF and waits out Unit's 30-second keepalive idle timeout. test_route_if was the visible cost: test_route_if_match took 120s and test_route_if_njs 60s on every full-suite leg. request_headers() now merges "Connection: close" into a caller's dict unless the caller set "Connection" itself, and close=False opts out for a request that means to keep the connection: the three websocket handshake tests that exercise an absent Connection, and test_chunked_pipeline, which pipelines two requests on one socket. Measured on a build of this tree: test/test_route_if.py 121.21s -> 1.36s. route_if, chunked, access-log and routing files pass locally (147 passed, 8 skipped); test_access_log_object_escape fails on this box with and without the change. Co-Authored-By: DeepSeek V4.1 Flash --- test/test_asgi_websockets.py | 1 + test/test_chunked.py | 1 + test/test_java_websockets.py | 1 + test/test_node_websockets.py | 1 + test/unit/http.py | 26 ++++++++++++++++++++++++-- 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index f034195e1..a4463e4db 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -293,6 +293,7 @@ def test_asgi_websockets_handshake_connection_absent(): # FAIL 'Sec-WebSocket-Protocol': 'chat', 'Sec-WebSocket-Version': 13, }, + connection_close=False, ) assert resp['status'] == 400, 'status' diff --git a/test/test_chunked.py b/test/test_chunked.py index ef449fab3..ea1743f70 100644 --- a/test/test_chunked.py +++ b/test/test_chunked.py @@ -56,6 +56,7 @@ def test_chunked_pipeline(): 'Host': 'localhost', 'Transfer-Encoding': 'chunked', }, + connection_close=False, body='1\r\n$\r\n0\r\n\r\n', ) diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 94ac6f864..031d6924f 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -217,6 +217,7 @@ def test_java_websockets_handshake_connection_absent(): # FAIL 'Sec-WebSocket-Protocol': 'chat', 'Sec-WebSocket-Version': 13, }, + connection_close=False, ) assert resp['status'] == 400, 'status' diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 68bdd578f..26baca07a 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -237,6 +237,7 @@ def test_node_websockets_handshake_connection_absent(): # FAIL 'Sec-WebSocket-Protocol': 'chat', 'Sec-WebSocket-Version': 13, }, + connection_close=False, ) assert resp['status'] == 400, 'status' diff --git a/test/unit/http.py b/test/unit/http.py index 84af79d57..650ceb1ce 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -11,6 +11,28 @@ from unit.option import option +def request_headers(headers=None, close=True): + """Return request headers with "Connection: close" merged in. + + HTTP/1.1 keeps a connection open by default and recvall() reads a + response to EOF, so a one-shot request that omits "Connection: close" + waits out the server's idle timeout. A "Connection" the caller set is + left alone; close=False drops the merge for a request that means to + keep the connection -- a handshake, a pipeline, a socket that is + reused. + """ + if headers is None: + headers = {'Host': 'localhost'} + + else: + headers = dict(headers) + + if close: + headers.setdefault('Connection', 'close') + + return headers + + class HTTP1: def http(self, start_str, **kwargs): sock_type = kwargs.get('sock_type', 'ipv4') @@ -18,8 +40,8 @@ def http(self, start_str, **kwargs): url = kwargs.get('url', '/') http = 'HTTP/1.0' if 'http_10' in kwargs else 'HTTP/1.1' - headers = kwargs.get( - 'headers', {'Host': 'localhost', 'Connection': 'close'} + headers = request_headers( + kwargs.get('headers'), kwargs.get('connection_close', True) ) body = kwargs.get('body', b'') From e459976f7754444daeb8aa2ebee01ac2b4f1d8ef Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 17:47:33 +0200 Subject: [PATCH 198/391] test: case-insensitive "Connection" and one knob name in request_headers request_headers() now recognises a "Connection" the caller set in any case, so a lowercase key cannot put two Connection headers on the wire, and its parameter takes the kwarg's name (connection_close) rather than a second spelling for the same knob. The docstring says what the merge does not do: an explicit dict gets no "Host", and a zero-header request is not expressible through the helper -- test_routes_match_host_empty asks for no Host and now sends "Connection: close" as well. The dict() copy that the merge needs also stops http() from writing "Content-Type" and "Content-Length" into the caller's own dict, which it did on the first body-carrying request that passed one. Co-Authored-By: DeepSeek V4.1 Flash --- test/unit/http.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test/unit/http.py b/test/unit/http.py index 650ceb1ce..1d4004662 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -11,15 +11,19 @@ from unit.option import option -def request_headers(headers=None, close=True): +def request_headers(headers=None, connection_close=True): """Return request headers with "Connection: close" merged in. HTTP/1.1 keeps a connection open by default and recvall() reads a response to EOF, so a one-shot request that omits "Connection: close" - waits out the server's idle timeout. A "Connection" the caller set is - left alone; close=False drops the merge for a request that means to - keep the connection -- a handshake, a pipeline, a socket that is - reused. + waits out the server's idle timeout. A "Connection" the caller set, in + any case, is left alone; connection_close=False drops the merge for a + request that means to keep the connection -- a handshake, a pipeline, a + socket that is reused. + + Only a caller that passes no dict at all gets "Host: localhost"; an + explicit dict is given "Connection: close" and nothing else, and a + zero-header request is not expressible through this helper. """ if headers is None: headers = {'Host': 'localhost'} @@ -27,8 +31,10 @@ def request_headers(headers=None, close=True): else: headers = dict(headers) - if close: - headers.setdefault('Connection', 'close') + if connection_close and not any( + name.lower() == 'connection' for name in headers + ): + headers['Connection'] = 'close' return headers From 586cf1350790a797868efe0d6a7a8387afff200b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 18:11:18 +0200 Subject: [PATCH 199/391] ci: spread the shared suite across the python legs Three Python versions each ran the whole `test` directory -- about 12 minutes a leg, twice what any other language costs -- although only the interpreter-specific suites need all three versions. Each Python leg now runs `test_python*`/`test_asgi*`/`test_wsgi*` whole and a round-robin third of the remaining files; `unit` still runs the whole tree in one piece. Both lists are built from the files that exist (`ls test/test_*.py`), never from a glob handed to pytest: there is no `test_wsgi_*.py` (WSGI is exercised through the python module in `test_python_*`), and an unmatched pattern survives shell expansion literally, so pytest aborts the whole leg with "file or directory not found" instead of running the rest. The runtime count and the shard index are validated as non-empty decimal integers (the count also non-zero) right after the jq calls. jq prints "null" for a version missing from pkg/eol.json, and a non-integer in the `if` comparisons fails open: the leg would run its interpreter files, skip every shared one, and still report green. Shared files run on one interpreter instead of three, and for the ~20 of them that declare a Python-module prerequisite the count is 3 -> 1 with no fallback: the `unit` leg does not build the module, so they self-skip there. Co-Authored-By: DeepSeek V4.1 Flash Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-test.yml | 77 ++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b5b3965f1..996a02871 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -97,12 +97,83 @@ jobs: # libunit port/fd lifecycle covered in the python leg only. testpath="test/test_${module}* test/test_app_lifecycle.py" - # Run all tests for "unit" and "python" - # Python is the default module for tests - if [ "$module" = "unit" ] || [ "$module" = "python" ]; then + # "unit" is the only leg that runs the language-agnostic suite in + # one piece; the Python-application tests self-skip there without + # the module, and every other test runs as written. + if [ "$module" = "unit" ]; then testpath="test" fi + # A Python leg runs its interpreter-specific suites whole -- that is + # what the three versions are here for -- and takes a round-robin + # share of the rest. Three versions each running the whole + # language-agnostic suite again cost about 12 minutes a leg, twice + # any other language, for tests no interpreter distinguishes. Each + # shared file now runs on one Python version instead of three; the + # union over the legs is unchanged. + # + # Both lists are built from the files that exist, never from a glob + # handed to pytest. There is no test_wsgi_*.py -- WSGI is exercised + # through the python module in test_python_* -- and an unmatched + # pattern survives shell expansion literally, so pytest aborts the + # whole leg with "file or directory not found" instead of running the + # rest. `test/test_*.py` is depth 1 while the unit leg runs the whole + # `test` tree, so a nested test file would run on unit and on no + # python leg; keep the two definitions of "the suite" in step. + if [ "$module" = "python" ]; then + versions=$(jq -r '.runtimes.python | length' pkg/eol.json) + shard=$(jq -r --arg v "${version}" \ + '[.runtimes.python[].version] | index($v)' pkg/eol.json) + + # A jq miss prints "null" rather than failing, and a non-integer + # in either comparison below fails open: the leg would run its + # interpreter files, skip every shared one, and still report + # green. Fail loudly instead, like the OpenSSL 4.0 assertion. + case "${versions}" in + ''|0|*[!0-9]*) + echo "::error::no python runtimes in pkg/eol.json" >&2 + exit 1 + ;; + esac + + case "${shard}" in + ''|*[!0-9]*) + echo "::error::no shard for python-${version} in pkg/eol.json" >&2 + exit 1 + ;; + esac + + files="" + whole="" + i=0 + for f in $(ls test/test_*.py | sort); do + case "$f" in + test/test_python*|test/test_asgi*|test/test_wsgi*) + whole="${whole} ${f}" + continue + ;; + test/test_app_lifecycle.py) + continue + ;; + esac + + if [ $((i % versions)) -eq "$shard" ]; then + files="${files} ${f}" + fi + + i=$((i + 1)) + done + + testpath="${files}${whole}" + + # app_lifecycle self-skips unbuilt runtimes, so one Python leg is + # enough to carry it; the other language legs still run it as + # before. + if [ "$shard" -eq 0 ]; then + testpath="${testpath} test/test_app_lifecycle.py" + fi + fi + echo "module=${module}" >> "$GITHUB_OUTPUT" echo "version=${version}" >> "$GITHUB_OUTPUT" echo "testpath=${testpath}" >> "$GITHUB_OUTPUT" From 8157a805ba8dfe7e1cf9a5fbbd9ae05885205cb8 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 19:11:57 +0200 Subject: [PATCH 200/391] ci(audit): audit every Cargo.lock, and close the advisories it finds The audit job only checked tools/unitctl/Cargo.lock, so the other seven tracked locks could carry an open advisory with CI green. That is what happened: four locks were failing an audit nobody was running. The new audit.yml discovers every tracked Cargo.lock and audits them as a matrix, so a lock added by a future tool is covered the moment it is committed, and every failing lock is reported rather than only the first. The discovery step refuses an empty list. A matrix over nothing would report success while auditing nothing. The weekly trigger moves with the audit. A new advisory against a lock nobody has touched is exactly the case it exists for. The unitctl workflow keeps building, testing and releasing; its unitctl-only audit job is subsumed. Locks fixed by this change: tools/unitctl rustls 0.23.42 -> 0.23.45 (RUSTSEC-2026-0285, GHSA-2mjx-qc3c-rqvc) with aws-lc-rs 1.17.1 -> 1.18.1, aws-lc-sys 0.42.0 -> 0.45.0 and rustls-webpki 0.103.13 -> 0.103.15, which the new rustls requires. rustls accepted TLS 1.3 handshake messages sent at the wrong encryption level when they followed a key-changing message in the same record, where RFC 8446 Sect. 5.1 requires the connection to be terminated. This is the advisory that had master's audit job red since 2026-09-14. src/otel, test/fake_otlp h2 0.4.14 -> 0.4.19 (RUSTSEC-2026-0258) anyhow 1.0.102 -> 1.0.104 (RUSTSEC-2026-0190) The two h2 lines below start from a different version than these two. The locks are independent and had drifted; both land on 0.4.19. src/wasm-wasi-component wasmtime 47.0.2 -> 47.0.4, with wiggle and cranelift (RUSTSEC-2026-0269, -0268, -0223, -0222; one is a filesystem sandbox escape) h2 0.4.15 -> 0.4.19 (RUSTSEC-2026-0258) chacha20 0.10.1 -> 0.10.2 (yanked) Every one of those is a version and checksum move: 223 packages before and after in the wasm lock, none added, none removed. Verified locally: cargo audit --deny warnings exits 0 on all eight locks and exited 1 on four of them before; cargo metadata --locked resolves each workspace without rewriting it; src/otel, test/fake_otlp and src/wasm-wasi-component pass cargo check. The musl artifact is unaffected: the feature-off graph is still 303 lines with no cc, cmake or aws-lc-sys. Co-Authored-By: DeepSeek V4.1 Flash --- .github/workflows/audit.yml | 82 ++++++++++++++++ .github/workflows/unitctl.yml | 34 ------- src/otel/Cargo.lock | 8 +- src/wasm-wasi-component/Cargo.lock | 152 ++++++++++++++--------------- test/fake_otlp/Cargo.lock | 8 +- tools/unitctl/Cargo.lock | 16 +-- 6 files changed, 174 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/audit.yml diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 000000000..b21dd7c98 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,82 @@ + +name: "Audit (cargo)" + +# Every tracked Cargo.lock is checked against the RustSec advisory database. +# The weekly trigger is what makes this a gate rather than a formality: a new +# advisory can land against a lock file nobody has touched, and without the +# schedule the audit would only run when a lock changes. +on: + pull_request: + paths: + - '**/Cargo.lock' + - '**/Cargo.toml' + # A change to this file must run this file. + - .github/workflows/audit.yml + push: + branches: + - master + paths: + - '**/Cargo.lock' + - '**/Cargo.toml' + - .github/workflows/audit.yml + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Cancel superseded runs for the same pull request. Non-PR events fall back + # to the run id, which is unique, so a push is never grouped with another: + # Actions keeps only one PENDING run per group and would drop a queued commit + # even with cancel-in-progress false, which would break bisectability. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + locks: + # The lock list is discovered instead of hard-coded, so a lock added by a + # future tool is covered the moment it is committed. Passing it to the + # audit job also makes that a matrix, so every failing lock is reported, + # not just the first. + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + locks: ${{ steps.list.outputs.locks }} + steps: + - uses: actions/checkout@v5 + + - name: List tracked lock files + id: list + run: | + locks=$(git ls-files '*Cargo.lock' \ + | jq -R -s -c 'split("\n") | map(select(length > 0))') + test "$(echo "$locks" | jq 'length')" -gt 0 + echo "locks=$locks" >> "$GITHUB_OUTPUT" + + audit: + needs: locks + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + lock: ${{ fromJSON(needs.locks.outputs.locks) }} + steps: + - uses: actions/checkout@v5 + + - run: rustup update stable + + # Prebuilt binary instead of `cargo install cargo-audit`, which would + # compile it from source on every run. + - name: Install cargo-audit + uses: taiki-e/install-action@v2 + with: + tool: cargo-audit + + # Without --deny warnings the command exits 0 on an unmaintained, + # unsound or yanked crate, so the job would pass while an advisory is + # open. + - name: Audit ${{ matrix.lock }} + run: cargo audit --deny warnings --file '${{ matrix.lock }}' diff --git a/.github/workflows/unitctl.yml b/.github/workflows/unitctl.yml index 752631a16..35099bc24 100644 --- a/.github/workflows/unitctl.yml +++ b/.github/workflows/unitctl.yml @@ -14,10 +14,6 @@ on: - master tags: - '[0-9]+.[0-9]+.[0-9]+' - schedule: - # The audit job needs a trigger that does not depend on a code change. A - # new advisory can land against a lock file nobody has touched. - - cron: "0 6 * * 1" workflow_dispatch: inputs: version: @@ -38,8 +34,6 @@ concurrency: jobs: test: - # The weekly trigger exists for the audit job alone. - if: github.event_name != 'schedule' runs-on: ${{ matrix.os }} timeout-minutes: 25 defaults: @@ -90,31 +84,6 @@ jobs: - name: Test ${{ matrix.os }} run: ${{ env.MAKE }} test - audit: - # Check Cargo.lock against the RustSec advisory database. This reads the - # lock file only, so it needs no openapi generation and no build. - runs-on: ubuntu-latest - timeout-minutes: 10 - defaults: - run: - working-directory: tools/unitctl - steps: - - uses: actions/checkout@v5 - - - run: rustup update stable - - # Prebuilt binary instead of `cargo install cargo-audit`, which would - # compile it from source on every run. - - name: Install cargo-audit - uses: taiki-e/install-action@v2 - with: - tool: cargo-audit - - # Without --deny warnings the command exits 0 on an unmaintained crate, - # so the job would pass while an advisory is open. - - name: Audit Cargo.lock - run: cargo audit --deny warnings - musl: # The only leg that compiles unitctl without the "tls" feature. The # feature-off code -- the #[cfg(not(feature = "tls"))] blocks and the test @@ -128,7 +97,6 @@ jobs: # *host* -- "make release" on Alpine -- is not built here. The make rule # for that host is checked below instead, which is the part that can be # checked without a musl runner. - if: github.event_name != 'schedule' runs-on: ubuntu-latest timeout-minutes: 30 defaults: @@ -209,8 +177,6 @@ jobs: | grep -q -- '--no-default-features' build: - # The weekly trigger exists for the audit job alone. - if: github.event_name != 'schedule' runs-on: ${{ matrix.os }} timeout-minutes: 30 defaults: diff --git a/src/otel/Cargo.lock b/src/otel/Cargo.lock index 365e14b77..5122b5f62 100644 --- a/src/otel/Cargo.lock +++ b/src/otel/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "async-trait" @@ -222,9 +222,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", diff --git a/src/wasm-wasi-component/Cargo.lock b/src/wasm-wasi-component/Cargo.lock index e219eaece..b7b2598cc 100644 --- a/src/wasm-wasi-component/Cargo.lock +++ b/src/wasm-wasi-component/Cargo.lock @@ -113,9 +113,9 @@ checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "cap-fs-ext" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78e5a3368ae89b7cb68186411452b4b9fac8b41be9c19bf3f47c2d2c8e36e6b" +checksum = "56ff379b70af8e08307a8f65e7040c7301cb4a572538ade16b4984f0da77847f" dependencies = [ "cap-primitives", "cap-std", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "cap-primitives" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdadbd7c002d3a484b35243669abdae85a0ebaded5a61117169dc3400f9a7ff0" +checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" dependencies = [ "ambient-authority", "fs-set-times", @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "cap-std" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7281235d6e96d3544ca18bba9049be92f4190f8d923e3caef1b5f66cfa752608" +checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8" dependencies = [ "cap-primitives", "io-extras", @@ -180,9 +180,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -238,27 +238,27 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f0435c6939d6ef218bf0a3063094f2cc5c46f7a44a88f81cf932da8d3922ff8" +checksum = "16c273ba1bfc3a1cb27cb4f83df27134f3cb638a61f5ed79ab22cf86eb13da0d" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aca52511cc21a2c9c927aef493d11c8087bdc5483249a3d93aee41e81b8bd4d1" +checksum = "cbf8909326a466739e83ffe11e74c0c8c6b4bfa911b612aece915daa746cff58" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6867e99f068454b0c81c50b807af88aa80aaf383105e2be26675d81fc4278466" +checksum = "3cb26b06b54d8b2f8cdf4d440a32fc3b3b91c71c162333a69a70a66fa265fc45" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -266,9 +266,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf66017aed971344230199ad35ca01c955cf824a0e65fed5240452d2a302445" +checksum = "7ce4e57dbb7f73d08011808a19787c3a03c3e7854b2a479ed0b788915fb671be" dependencies = [ "serde", "serde_derive", @@ -277,9 +277,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "933b2e5237665bf9367a34ad2100eed70c94b2796586cb09d9ba11693e061667" +checksum = "74b0a8968f33d0bc13bc86fe70dd3947e93acc9f358ed9848e9988448972a9be" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -308,9 +308,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322e3ca0603683cc17f0c808559c0fb85ad52b0f20ace0478bdd4ba2fa1e9678" +checksum = "f69e8e2fe9deb48ef7b8d0c61612f8c1e2277d0421c201e30a4c844a4ebac698" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -321,24 +321,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287ab88211d049178dc8dda243e2db176919d8b29db605c0efc7c6d76ebea0f7" +checksum = "ad4db1e87a65e9c5a832e3ede160830d4e34df938ac38b5604819f8825687e73" [[package]] name = "cranelift-control" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11438f9becf9a1e1e061bf76a33241b941a07ed01a489e9dad5fa79dc3624e9" +checksum = "9248cd37bb6ec460981f57ead368e32b1eca7a207ad50e3e9c7adc3b161f20e9" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "584ecabe1462593a9cba9efbf9022e0987421ddb548989b455ab8bdd444b743c" +checksum = "26b8bc91c0a3530d132acbf118965dabe9a084029eb0fa74b64760360dec3316" dependencies = [ "cranelift-bitset", "serde", @@ -348,9 +348,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34249873145213a12106d5fd8a98a9dc9e90e16525e2337b2907874d1e7e9ee7" +checksum = "5086f2ebc084a98b387e522680f62ef65a512444d8382151e03088716d3a1714" dependencies = [ "cranelift-codegen", "hashbrown 0.17.1", @@ -361,15 +361,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f1c1dc4d6b4bc4f0be12b08cee0e59f7610287827aa62590f8e9ec6f45dff4" +checksum = "b6a658670f779afc2df083be201ebe54805f69b7b9ffa8dab04142b72ed751c9" [[package]] name = "cranelift-native" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31d9e4f6d9a3777b1ffa818000b4a6d7afd1b8787af71a9c58d94baef9076ced" +checksum = "16a41083b1fd953debd6f32d21fcda765b258f323ce907fdd6b4715f2769c478" dependencies = [ "cranelift-codegen", "libc", @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.134.2" +version = "0.134.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4155f8a1afdef96a0272c67804926eee4ef35f9708ab22ad6878d4c9543df596" +checksum = "244d9cd0759b0a3ccdd587cafb13434f9ea43b5c9d07701d277daf5cd0f7b7c1" [[package]] name = "crc32fast" @@ -633,9 +633,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -1091,9 +1091,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6064e52588328c1275d690baad99f8d4b3fdd870736966f1a63cfcbb20560b6" +checksum = "ed2bd641cb6a3ec5bffe60405568933163e0ea619da0943e9d2a417ceaa7fe82" dependencies = [ "cranelift-bitset", "log", @@ -1103,9 +1103,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e832980317bfc7f8c8538ffc9c69b82912f3e33ec70f7c855d521654c70fa62" +checksum = "d3fd4024d1e559eaf5579bc8ad84a2b5915f34b47acc13a06ffba7dac401b166" dependencies = [ "proc-macro2", "quote", @@ -1588,9 +1588,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06d9f263ccb212017dae1d109b9997f218a7675db1e9a7bedf07ae1643322bf3" +checksum = "4cdaf5b8af5713146e3a62d940c71e3eb2ad1ebacfa6b55f0c8509a5b4b87718" dependencies = [ "addr2line", "async-trait", @@ -1630,9 +1630,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9915e3d90c36a16e0fec8be746ee87e21cf0d4a017ecf09c49b7d1cc6f31b2b6" +checksum = "58793271eab7ec26bab99ca497131f8b37702e7dc39e2f2d18293e3e915b4964" dependencies = [ "anyhow", "cpp_demangle", @@ -1661,9 +1661,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b627ea698db36fb9110aa77868467f1d23791a76a88a433cb6a43e1242073b77" +checksum = "c5bba4eff4804620bae04d7793308d97a0c5a8a97ea19e635452518f84560f84" dependencies = [ "anyhow", "proc-macro2", @@ -1676,15 +1676,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f45bf943cff1d2f1976bac6a241ca4ecaf29a6efd5f1ad89e7722439fa5c392f" +checksum = "3d7032ff6b4d7a45e05be1cb32ec1756df5f2669f08296dbe2c61c3789dead2c" [[package]] name = "wasmtime-internal-core" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea33e1f9685be82f90223bebcf2ffefb6b596ea3d82a3c42384527b9c6f5b447" +checksum = "f41d3b2cb9c3de7b696690af070d408b5e357011832f8b8bb7fbb315dbb620ed" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -1694,9 +1694,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1ce104719932a48dbdfec0cc292c96494b0bfe764946e45fbfd3340e3a68f9" +checksum = "1d5cda5b90247978dd2ba6ca4e309508da19e9244445671d2d0e6a34e5c3b847" dependencies = [ "cfg-if", "cranelift-codegen", @@ -1721,9 +1721,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976e4b71b9c0da8a9a7bb8d5fa0abedc4da879b397111fd91ece73be524fe621" +checksum = "4d126366176553e8f304e8ac2078a4cc37134fe8f19d4d4695a09c5a2f577f10" dependencies = [ "cc", "cfg-if", @@ -1736,9 +1736,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f60ecba3497cfc26284903c2e79023fd5ea991b95b0e8dafe994fd9fca4c4e" +checksum = "c1d28ff131bc40e11d84bc3a9fa0d264506439ffbbc89f3562fbb529d1caa617" dependencies = [ "cc", "wasmtime-internal-versioned-export-macros", @@ -1746,9 +1746,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fae6decb025ae6d12f3391b36deb62a6838d45b18864f796d84bced722ceac0d" +checksum = "4a52de8b6afbfcd618073f592c1450cb661574e7061f3f4a44f1f7ec0c7f7909" dependencies = [ "cfg-if", "libc", @@ -1758,9 +1758,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0982122d9470d3d72ed7aa9ed91e69538ae44c1494cd22d78fec7846b248a940" +checksum = "5524fbaaf3d2134dd6229164d7fc81e443cce06d1e0576524d5f4e86e747cb13" dependencies = [ "cfg-if", "cranelift-codegen", @@ -1771,9 +1771,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "363cf88a80e6ed13311d0d34d9bcd02a2e07d98113b994ff41ed386a206f498e" +checksum = "e93bb9ae112a80618510f938275a5d88637b656e7d0a021fc906f52206c708ad" dependencies = [ "proc-macro2", "quote", @@ -1782,9 +1782,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ff41bb1656cbeb7befc42b8a7078b30a5f8393a0c8bca36a6cec94360c1a16" +checksum = "bfdbdd9d6fab1ecb67a7dc189e32c9e341444945fb709437bf8aab25d9a88459" dependencies = [ "anyhow", "bitflags", @@ -1795,9 +1795,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1750b8e5e5d5b1ee3c0eb602d2c49d6a37a36e9996eaf0708b48ac92b62b06" +checksum = "b373095a6dc8382da56882ad16d7ea16771dd6927dd480b1ce3fe7c4b816d2cf" dependencies = [ "async-trait", "bitflags", @@ -1821,9 +1821,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-http" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014b8f05cacb7a5d665536e8a733b887bd192a9c179037580d48d0a6a82ecd34" +checksum = "372fc7845f26cd8f69bcbf6e4234fcc032f718480c7722fa20a28eb74d58998e" dependencies = [ "async-trait", "bytes", @@ -1841,9 +1841,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-io" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54777d3afeaf3f32cc444df661d9cb4fd57067971e183c2b1f5251f33b939e68" +checksum = "51b3c9b39b1a1b3029f80e614623e182f165133dfaca8610f156450343f080a4" dependencies = [ "async-trait", "bytes", @@ -1863,9 +1863,9 @@ dependencies = [ [[package]] name = "wiggle" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a23c2dc694dfafb35beb542e172d7c256488912cd84902828f8661723ad0eb7" +checksum = "8c08bd1d66a10c7d4deaeb746cb1c5b5c62fb631907f18dfe058ecba71c99002" dependencies = [ "bitflags", "thiserror 2.0.18", @@ -1877,9 +1877,9 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fe48e5e455e827339d8e65de602cc7839d241bd7063be1fc235e35581788ad" +checksum = "19eb81a4ceda41cc6fa2adc8e865473fc2e85630149ed7f5d95356b0010f0ed3" dependencies = [ "heck", "proc-macro2", @@ -1891,9 +1891,9 @@ dependencies = [ [[package]] name = "wiggle-macro" -version = "47.0.2" +version = "47.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb520c20bed78e5a311d4c9ad883ac3b6d070617d07a19fbe5dc0b408c501b4b" +checksum = "c0e0d7c6d28846ffa438e421976734a2057f13a7e20af9f2a8607a6714e1e358" dependencies = [ "proc-macro2", "quote", diff --git a/test/fake_otlp/Cargo.lock b/test/fake_otlp/Cargo.lock index 7a15315cd..ba7a103c0 100644 --- a/test/fake_otlp/Cargo.lock +++ b/test/fake_otlp/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "async-trait" @@ -197,9 +197,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index a210dbec9..2244999cf 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -69,9 +69,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -79,9 +79,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -1050,9 +1050,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -1086,9 +1086,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", From 641ecbfd9e8744a2d488460e83262a4214f0220e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 19:26:54 +0200 Subject: [PATCH 201/391] docs(eol): record both wasmtime pins, and which one is audited FreeUnit pins wasmtime twice. pkg/contrib builds 43.0.1 from a source tarball for the wasm module's C API; src/wasm-wasi-component/Cargo.lock carries the Rust crate, 47.0.4 after #399. Neither appeared in EOL.md or pkg/eol.json, which is where build-time and bundled dependencies are documented, so a four-major gap between them was visible nowhere. It is not only a documentation gap. cargo audit reads Cargo.lock files, so the crate is checked on every run and the tarball is checked by nothing -- and 43.0.1 is affected by RUSTSEC-2026-0269, a filesystem sandbox escape, and RUSTSEC-2026-0222. Both list patches per release branch, skip 43.x, and declare unaffected = []. That is issue #400; this commit only writes down what is pinned and what covers it. Add a rule with it: a dependency pinned twice gets a row per pin, naming the check that reaches it. The gap here was invisible because one row would have described both pins and neither accurately. The new keys are documentation. pkg/eol/src/main.rs reads _grace_*, .os and .runtimes, and its Config struct does not deny unknown fields; the checker still exits 0 on the edited file with the same 13 warnings as before. Co-Authored-By: Claude Opus 5 (1M context) --- EOL.md | 6 ++++++ pkg/eol.json | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/EOL.md b/EOL.md index 4dbcf1266..85a504430 100644 --- a/EOL.md +++ b/EOL.md @@ -92,6 +92,8 @@ needs an upgrade or replacement decision instead. | Eclipse Jetty | `jetty-util` / `jetty-server` / `jetty-http` jars bundled by the Java module | 9.4.58.v20250814 | **Aug 2025 (EOL)** † | Jetty 9.4 lost community support on 2025-08-14 and the bundled build is its last release. Needs an upgrade-or-replace decision — Jetty 10/11 are EOL too; 12.x is the supported line. | | Eclipse ECJ (JDT batch compiler) | JSP compilation jar bundled by the Java module | 3.26.0 | none published | Tracks Eclipse releases; endoflife.date has no product for it. Pinned build is from Jun 2021; current is 3.42.0 (Jun 2025). | | ClassGraph | classpath scanning jar bundled by the Java module | latest | — | No upstream EOL schedule; pinned and bumped as needed. | +| Wasmtime (C API) | WebAssembly runtime built by `pkg/contrib` and linked by the `wasm` module | 43.0.1 | none published | Fetched as a source tarball (`pkg/contrib/src/wasmtime/version`), so `cargo audit` never sees it. Four majors behind the Rust pin below, and affected by RUSTSEC-2026-0269 and -0222 — see #400. | +| Wasmtime (Rust crate) | WebAssembly runtime used by the `wasm-wasi-component` module | 47.0.4 | none published | `src/wasm-wasi-component/Cargo.lock`, audited by the `Audit (cargo)` workflow on every lock. Bumped with the advisories it closes. | ## Rules @@ -107,6 +109,10 @@ needs an upgrade or replacement decision instead. - **Dependencies:** the floors in the Dependency Support section are not part of the EOL + grace policy; they track what FreeUnit builds and bundles, not what it ships as a variant. +- **A dependency pinned twice needs both rows.** Wasmtime is pinned once as a Rust + crate and once as a source tarball, and only the crate is reachable by `cargo + audit`. List every pin separately and say which one an automated check covers, so + a gap between them is visible here rather than only in a lock file. - **LTS OS (Ubuntu, RHEL, Debian):** 3-year extension applies to standard EOL, not extended security maintenance dates. diff --git a/pkg/eol.json b/pkg/eol.json index a0d997548..62ef44d3b 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -126,6 +126,20 @@ "version": "latest", "note": "No floor and no EOL schedule upstream: pinned to the current release and bumped as needed.", "source": "https://github.com/classgraph/classgraph/releases" + }, + "wasmtime_c_api": { + "role": "WebAssembly runtime built by pkg/contrib and linked by the wasm module (pkg/contrib/src/wasmtime/version, WASMTIME_VERSION)", + "version": "43.0.1", + "eol": null, + "eol_note": "Fetched as a source tarball from packages.freeunit.org, so cargo audit never sees it: it reads Cargo.lock files only. Four majors behind the wasmtime_crate pin, and affected by RUSTSEC-2026-0269 (filesystem sandbox escape) and RUSTSEC-2026-0222 (stores mix up type indices) -- both list per-branch patches that skip 43.x and declare unaffected = []. Tracked in freeunitorg/freeunit#400.", + "source": "https://github.com/bytecodealliance/wasmtime/releases" + }, + "wasmtime_crate": { + "role": "WebAssembly runtime used by the wasm-wasi-component module (src/wasm-wasi-component/Cargo.lock)", + "version": "47.0.4", + "eol": null, + "eol_note": "Audited by the Audit (cargo) workflow, which checks every tracked Cargo.lock against the RustSec database, so an advisory here arrives as a red job rather than silence. Bumped with the advisories it closes.", + "source": "https://github.com/bytecodealliance/wasmtime/releases" } } } From befc02cade33890a1aea54ead202bf533815b4ab Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 19:50:35 +0200 Subject: [PATCH 202/391] test: read one frame in test_asgi_websockets_large, and check it The test read two frames for a message the mirror app echoes as one, so the second read waited 30 seconds on every run. It still passed: frame_read() fills frame['data'] whatever the opcode is (test/unit/applications/websockets.py:136), the frame that eventually arrived carried no payload, and "data += b''" left the comparison true. That frame is the websocket keepalive PING. A websocket connection is governed by websocket_conf.read_timeout, 60s, and websocket_conf.keepalive_interval, 30s (src/nxt_router.c:2903-2904); the listener idle_timeout at :2891 does not apply to it. After 30s idle, nxt_h1p_conn_ws_keepalive() sends a frame built with nxt_websocket_frame_init(wsh, 0), so its payload is empty. A close frame would have made the old test fail rather than pass: the going-away close carries a 2-byte code and "Remote peer is going away", and that text would have been concatenated onto data. Drop the second read and use check_frame(), the helper this file already uses in its fragmentation and autobahn tests. It asserts fin and opcode as well as the payload, so a control frame can no longer satisfy the comparison by being empty. Dropping the read alone would have made the test fast without making it strict -- it passed before and after -- and the frame assertion is what gives the change something to prove. One read is right here and two are right in test_node_websockets_large, which is where this test's shape probably came from. That one loads websockets/mirror_fragmentation, whose app.js omits the fragmentOutgoingMessages and fragmentationThreshold overrides that websockets/mirror sets, so the node library really does fragment its echo. Recorded in the test so nobody removes a read there too. Measured in a container on ubuntu:24.04: echoes of 3000, 70000, 600000 and 1048512 bytes each came back as a single fin=1 frame in under 0.01s with no trailing bytes, and the unpatched test reproduced its stall at 30.11s. Above max_frame_size (1 MiB, src/nxt_router.c:2902) the server rejects rather than fragments. Co-Authored-By: Claude Opus 5 (1M context) --- test/test_asgi_websockets.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index a4463e4db..73102fa76 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -212,13 +212,21 @@ def test_asgi_websockets_large(): ws.frame_write(sock, ws.OP_TEXT, message) + # check_frame() asserts fin and opcode as well as the payload. This + # test used to read a second frame and concatenate it, which is how it + # sat for 30s on every run: frame_read() fills frame['data'] whatever + # the opcode is, the frame that eventually arrived was the zero-length + # keepalive PING (websocket_conf.keepalive_interval, + # src/nxt_router.c:2904), and appending b'' left the comparison true. + # + # The mirror echoes a message as one frame, so one read is right here. + # test_node_websockets_large reads twice on purpose: it loads + # websockets/mirror_fragmentation, whose app.js omits the + # fragmentOutgoingMessages and fragmentationThreshold overrides that + # websockets/mirror sets, so that server really does fragment. frame = ws.frame_read(sock) - data = frame['data'].decode('utf-8') - frame = ws.frame_read(sock) - data += frame['data'].decode('utf-8') - - assert message == data, 'large' + check_frame(frame, True, ws.OP_TEXT, message) sock.close() From 79f99c4537783423d04e68aefaedc34c3f2a0abf Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 16:40:50 +0200 Subject: [PATCH 203/391] port: a queued message owns the descriptors it sends nxt_port_send_port() hands another port's pair[1] and queue_fd to nxt_port_socket_write2() as bare numbers; NEW_PORT does not set NXT_PORT_MSG_CLOSE_FD, so the message borrows them. When the peer's socket is not writable nxt_port_msg_chk_insert() copies the message to the heap and queues it, and the sendmsg() that puts the descriptors into SCM_RIGHTS runs later from nxt_port_write_msgs(). If the owner closes them first -- nxt_port_close() does when a port goes away -- the deferred send names a closed number, or whatever descriptor the kernel has since put at that number. nxt_port_msg_alloc() now duplicates a borrowed fd[0]/fd[1] into the heap copy and marks it close_fd, so the existing paths close them exactly once: after the send, or from nxt_port_error_handler() and nxt_port_socket_cancel() when the message is dropped. A dup failure frees the copy and answers NULL, the same NXT_ERROR the caller already handles for an allocation failure, with its originals untouched. The inline stack copy is never duplicated, so the NXT_ERROR contract of the inline write is unchanged. A queued message now holds a descriptor per borrowed one until it is sent; port->messages has no bound. A bound is a separate change. Test: nxt_port_queued_fd_test drives the queued branch with write_ready clear, closes the originals, reopens a decoy at the reused number, then lets the write handler send over a socketpair. Before this change the peer receives the decoy. Closes #388. Co-Authored-By: Claude Opus 5 --- CHANGES | 7 + auto/sources | 1 + docs/changes.xml | 11 + src/nxt_port_socket.c | 76 ++++ src/test/nxt_port_queued_fd_test.c | 579 +++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 7 files changed, 679 insertions(+) create mode 100644 src/test/nxt_port_queued_fd_test.c diff --git a/CHANGES b/CHANGES index 175fd9a7f..2ee432521 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 *) Change: upgrade contrib njs to 1.0.1. + *) Bugfix: a port message that was queued because the peer's socket was + not writable borrowed the descriptors it carried -- for NEW_PORT, + another port's socket and queue -- so a port closed before the queue + drained left the deferred sendmsg() naming a closed number, or an + unrelated descriptor that had since reused it. The queued copy now + owns duplicates and closes them after the send (#388). + *) Bugfix: a static response that was subject to content negotiation now carries "Vary: Accept-Encoding". Without it a shared cache could store whichever representation it saw first and serve it to every client -- diff --git a/auto/sources b/auto/sources index ec2ab33fe..6d5c873a5 100644 --- a/auto/sources +++ b/auto/sources @@ -185,6 +185,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_ctrunc_test.c \ src/test/nxt_port_fd_test.c \ src/test/nxt_port_rpc_fd_test.c \ + src/test/nxt_port_queued_fd_test.c \ " diff --git a/docs/changes.xml b/docs/changes.xml index 7bd172beb..dbda4e5cd 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -31,6 +31,17 @@ replaces anything that is not valid UTF-8 with U+FFFD before it serializes. + + +a port message that was queued because the peer's socket was not writable +borrowed the descriptors it carried -- for NEW_PORT, another port's socket +and queue -- so a port closed before the queue drained left the deferred +sendmsg() naming a closed number, or an unrelated descriptor that had since +reused it. The queued copy now owns duplicates and closes them after the +send (#388). + + + upgrade contrib njs to 1.0.1. diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 91f29528b..0c99a9453 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -20,6 +20,7 @@ static uint8_t nxt_port_enqueue_buf(nxt_task_t *task, nxt_port_msg_t *pm, static nxt_int_t nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, nxt_port_send_msg_t *msg); static nxt_port_send_msg_t *nxt_port_msg_alloc(const nxt_port_send_msg_t *m); +static nxt_int_t nxt_port_msg_dup_fds(nxt_port_send_msg_t *msg); static nxt_int_t nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, nxt_bool_t *send_failed); static void nxt_port_write_handler(nxt_task_t *task, void *obj, void *data); @@ -519,10 +520,85 @@ nxt_port_msg_alloc(const nxt_port_send_msg_t *m) msg->allocated = 1; + /* + * A queued message must own the descriptors it names. + * + * The caller's copy borrows them: NEW_PORT names another port's pair[1] + * and queue_fd, START_PROCESS the application's shared port. The + * sendmsg() that puts them into SCM_RIGHTS runs later, from + * nxt_port_write_msgs(), and by then the owner may have closed them -- + * nxt_port_close() (src/nxt_port.c) does when a port goes away -- and + * the kernel may have given the numbers to something else. The deferred + * send would then fail with EBADF, or hand the peer whatever descriptor + * now sits at that number. + * + * So duplicate them here, and mark the copy as owning what it sends. + * From here on the send path closes them the way it closes any owned + * descriptor: after the send in nxt_port_write_msgs(), or from + * nxt_port_error_handler() and nxt_port_socket_cancel() when the message + * is dropped instead. The originals stay with their owner. + * + * The dup holds the descriptor until the message goes out, so a peer + * that stops reading holds this side's descriptor table open in + * proportion to what it was sent; port->messages has no bound. + */ + + if (!msg->close_fd && nxt_slow_path(nxt_port_msg_dup_fds(msg) != NXT_OK)) { + nxt_free(msg); + return NULL; + } + return msg; } +/* + * Replace a borrowed fd[0]/fd[1] with duplicates the message owns. On + * failure nothing is changed: a first duplicate is closed again, so the + * caller keeps exactly its originals. + */ + +static nxt_int_t +nxt_port_msg_dup_fds(nxt_port_send_msg_t *msg) +{ + nxt_fd_t fd0, fd1; + + if (msg->fd[0] == -1 && msg->fd[1] == -1) { + return NXT_OK; + } + + fd0 = -1; + fd1 = -1; + + if (msg->fd[0] != -1) { + fd0 = fcntl(msg->fd[0], F_DUPFD_CLOEXEC, 0); + if (nxt_slow_path(fd0 == -1)) { + nxt_thread_log_alert("dup(%FD) failed %E", msg->fd[0], nxt_errno); + return NXT_ERROR; + } + } + + if (msg->fd[1] != -1) { + fd1 = fcntl(msg->fd[1], F_DUPFD_CLOEXEC, 0); + if (nxt_slow_path(fd1 == -1)) { + nxt_thread_log_alert("dup(%FD) failed %E", msg->fd[1], nxt_errno); + + if (fd0 != -1) { + nxt_fd_close(fd0); + } + + return NXT_ERROR; + } + } + + msg->fd[0] = fd0; + msg->fd[1] = fd1; + msg->close_fd = 1; + + return NXT_OK; +} + + static void nxt_port_fd_block_write(nxt_task_t *task, nxt_port_t *port, void *data) { diff --git a/src/test/nxt_port_queued_fd_test.c b/src/test/nxt_port_queued_fd_test.c new file mode 100644 index 000000000..a99d63154 --- /dev/null +++ b/src/test/nxt_port_queued_fd_test.c @@ -0,0 +1,579 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for descriptor ownership of a queued port message + * (nxt_port_msg_alloc() in src/nxt_port_socket.c, #388). + * + * nxt_port_send_port() (src/nxt_port.c) hands another port's pair[1] and + * queue_fd to nxt_port_socket_write2() as bare numbers, without + * NXT_PORT_MSG_CLOSE_FD: the message borrows them. When the socket is not + * writable the message is copied to the heap and queued, and the sendmsg() + * that puts the descriptors into SCM_RIGHTS runs later. If the owner closes + * them first, the deferred send names a closed number -- or, once the kernel + * has reused it, an unrelated descriptor of this process. + * + * Two legs. The first inspects the queued copy: it must own duplicates, + * not the caller's numbers, and dropping it through the error handler must + * close exactly those duplicates. The second drives the race the issue + * describes end to end over a real socketpair: queue, close the originals, + * reopen something else at the same number, then let the write handler + * send. The peer must receive the descriptor that was queued, not the + * one that took its number. + */ + +#include +#include +#include +#include "nxt_tests.h" + +#include +#include +#include + + +static nxt_int_t nxt_port_queued_fd_test_owned(nxt_thread_t *thr); +static nxt_int_t nxt_port_queued_fd_test_reused(nxt_thread_t *thr); +static nxt_port_t *nxt_port_queued_fd_test_port(nxt_task_t *task, + nxt_event_engine_t *engine); +static void nxt_port_queued_fd_test_stub(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); +static void nxt_port_queued_fd_test_completion(nxt_task_t *task, void *obj, + void *data); +static void nxt_port_queued_fd_test_drain_wq(nxt_work_queue_t *wq); +static nxt_bool_t nxt_port_queued_fd_test_same_file(nxt_fd_t a, + const struct stat *b); +static nxt_fd_t nxt_port_queued_fd_test_recv_fd(nxt_fd_t sock); + + +static nxt_uint_t nxt_port_queued_fd_test_completions; + + +nxt_int_t +nxt_port_queued_fd_test(nxt_thread_t *thr) +{ + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port queued fd test started"); + + if (nxt_port_queued_fd_test_owned(thr) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_port_queued_fd_test_reused(thr) != NXT_OK) { + return NXT_ERROR; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port queued fd test passed"); + + return NXT_OK; +} + + +static void +nxt_port_queued_fd_test_stub(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + /* The fixture engine polls nothing. */ +} + + +/* + * A port on the caller's engine with a stub event interface, so that the + * write path's re-arm and block calls land on nothing. socket.write_ready + * is left clear: nxt_port_msg_chk_insert() then queues every message. + */ + +static nxt_port_t * +nxt_port_queued_fd_test_port(nxt_task_t *task, nxt_event_engine_t *engine) +{ + nxt_port_t *port; + + port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(port == NULL)) { + return NULL; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + port->socket.task = task; + port->socket.log = task->log; + port->socket.write_ready = 0; + port->socket.write = NXT_EVENT_INACTIVE; + port->engine = engine; + port->max_size = 1024; + port->max_share = 1024; + + port->socket.write_work_queue = &engine->fast_work_queue; + + return port; +} + + +static nxt_int_t +nxt_port_queued_fd_test_owned(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_fd_t fd0, fd1; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + struct stat st0, st1; + nxt_event_engine_t engine, *saved_engine; + nxt_port_send_msg_t *msg; + nxt_event_interface_t stub; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + fd0 = -1; + fd1 = -1; + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_queued_fd_test_stub; + stub.block_write = nxt_port_queued_fd_test_stub; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + thr->engine = saved_engine; + return NXT_ERROR; + } + + port = nxt_port_queued_fd_test_port(task, &engine); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + thr->engine = saved_engine; + return NXT_ERROR; + } + + fd0 = open("/dev/null", O_RDONLY); + fd1 = open("/dev/zero", O_RDONLY); + + if (fd0 == -1 || fd1 == -1 || fstat(fd0, &st0) != 0 + || fstat(fd1, &st1) != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: failed to open /dev/null"); + goto done; + } + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_queued_fd_test_completion; + nxt_port_queued_fd_test_completions = 0; + + /* The shape of nxt_port_send_port(): two borrowed descriptors. */ + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd0, fd1, + 0, 0, buf) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: write was not queued"); + goto done; + } + + if (nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: nothing queued"); + goto done; + } + + msg = nxt_queue_link_data(nxt_queue_first(&port->messages), + nxt_port_send_msg_t, link); + + if (!msg->close_fd) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the queued message does not own " + "its descriptors"); + goto done; + } + + if (msg->fd[0] == fd0 || msg->fd[1] == fd1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the queued message names the " + "caller's descriptors %FD %FD", msg->fd[0], msg->fd[1]); + goto done; + } + + if (!nxt_port_queued_fd_test_same_file(msg->fd[0], &st0) + || !nxt_port_queued_fd_test_same_file(msg->fd[1], &st1)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the queued duplicates name " + "other files"); + goto done; + } + + /* The owner closes; the queued copy must be unaffected. */ + + nxt_fd_close(fd0); + nxt_fd_close(fd1); + + if (!nxt_test_fd_is_open(msg->fd[0]) || !nxt_test_fd_is_open(msg->fd[1])) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: closing the originals closed " + "the queued copy"); + fd0 = -1; + fd1 = -1; + goto done; + } + + fd0 = msg->fd[0]; + fd1 = msg->fd[1]; + + /* Drop the queue: the duplicates go with it, and nothing else does. */ + + nxt_port_test_run_error_handler(task, port); + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + if (nxt_test_fd_is_open(fd0) || nxt_test_fd_is_open(fd1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: dropping the queue leaked the " + "duplicates"); + goto done; + } + + fd0 = -1; + fd1 = -1; + + if (nxt_port_queued_fd_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui completions, expected 1", + nxt_port_queued_fd_test_completions); + goto done; + } + + ret = NXT_OK; + +done: + + if (fd0 != -1 && nxt_test_fd_is_open(fd0)) { + nxt_fd_close(fd0); + } + + if (fd1 != -1 && nxt_test_fd_is_open(fd1)) { + nxt_fd_close(fd1); + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_port_test_run_error_handler(task, port); + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + } + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + thr->engine = saved_engine; + + return ret; +} + + +/* + * The race itself: the owner closes after the message was queued, the + * number is taken by something else, then the send goes out. + */ + +static nxt_int_t +nxt_port_queued_fd_test_reused(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_fd_t fd, decoy, got, pair[2]; + nxt_buf_t *buf; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + struct stat st, dst; + nxt_event_engine_t engine, *saved_engine; + nxt_event_interface_t stub; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + fd = -1; + got = -1; + decoy = -1; + pair[0] = -1; + pair[1] = -1; + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_queued_fd_test_stub; + stub.block_write = nxt_port_queued_fd_test_stub; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + thr->engine = saved_engine; + return NXT_ERROR; + } + + port = nxt_port_queued_fd_test_port(task, &engine); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + thr->engine = saved_engine; + return NXT_ERROR; + } + + /* SOCK_DGRAM: the type src/nxt_socketpair.c selects. */ + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: socketpair failed"); + goto done; + } + + if (nxt_slow_path(fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) { + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + + /* Installs the real write and error handlers; then not writable yet. */ + + nxt_port_write_enable(task, port); + + port->socket.write_ready = 0; + port->socket.write = NXT_EVENT_INACTIVE; + + fd = open("/dev/null", O_RDONLY); + if (fd == -1 || fstat(fd, &st) != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: failed to open /dev/null"); + goto done; + } + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_queued_fd_test_completion; + nxt_port_queued_fd_test_completions = 0; + + /* write_ready is clear: queued, not sent. */ + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + 0, 0, buf) + != NXT_OK + || nxt_queue_is_empty(&port->messages)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: write was not queued"); + goto done; + } + + /* + * The owner goes away, and the lowest free number -- the one just + * closed -- is taken by an unrelated file before the queue drains. + */ + + nxt_fd_close(fd); + + decoy = open("/dev/zero", O_RDONLY); + if (decoy == -1 || fstat(decoy, &dst) != 0) { + fd = -1; + goto done; + } + + if (decoy != fd) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the closed number %FD was not " + "reused (%FD), cannot stage the race", fd, decoy); + fd = -1; + goto done; + } + + fd = -1; + + /* Now the socket is writable and the queue drains. */ + + port->socket.write_ready = 1; + + port->socket.write_handler(task, &port->socket, NULL); + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + got = nxt_port_queued_fd_test_recv_fd(pair[0]); + if (got == -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the peer received no descriptor"); + goto done; + } + + if (nxt_port_queued_fd_test_same_file(got, &dst)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the peer received the descriptor " + "that reused the number, not the one that was queued"); + goto done; + } + + if (!nxt_port_queued_fd_test_same_file(got, &st)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the peer received an unexpected " + "descriptor"); + goto done; + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the message was sent but stayed " + "queued"); + goto done; + } + + if (nxt_port_queued_fd_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui completions, expected 1", + nxt_port_queued_fd_test_completions); + goto done; + } + + ret = NXT_OK; + +done: + + if (fd != -1) { + nxt_fd_close(fd); + } + + if (got != -1) { + nxt_fd_close(got); + } + + if (decoy != -1) { + nxt_fd_close(decoy); + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_port_test_run_error_handler(task, port); + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + if (pair[0] != -1) { + nxt_fd_close(pair[0]); + } + + if (pair[1] != -1) { + nxt_fd_close(pair[1]); + } + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + thr->engine = saved_engine; + + return ret; +} + + +static void +nxt_port_queued_fd_test_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_queued_fd_test_completions++; +} + + +static void +nxt_port_queued_fd_test_drain_wq(nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *t; + nxt_work_handler_t handler; + + while (wq->head != NULL) { + handler = nxt_work_queue_pop(wq, &t, &obj, &data); + handler(t, obj, data); + } +} + + +static nxt_bool_t +nxt_port_queued_fd_test_same_file(nxt_fd_t a, const struct stat *b) +{ + struct stat st; + + if (a == -1 || fstat(a, &st) != 0) { + return 0; + } + + return st.st_dev == b->st_dev && st.st_ino == b->st_ino + && st.st_rdev == b->st_rdev; +} + + +/* One datagram with its SCM_RIGHTS payload; -1 if none arrived. */ + +static nxt_fd_t +nxt_port_queued_fd_test_recv_fd(nxt_fd_t sock) +{ + ssize_t n; + nxt_fd_t fd; + struct iovec iov; + struct msghdr mh; + struct cmsghdr *cm; + u_char payload[256]; + union { + struct cmsghdr align; + u_char space[CMSG_SPACE(sizeof(int) * 2)]; + } cbuf; + + iov.iov_base = payload; + iov.iov_len = sizeof(payload); + + nxt_memzero(&mh, sizeof(mh)); + mh.msg_iov = &iov; + mh.msg_iovlen = 1; + mh.msg_control = &cbuf; + mh.msg_controllen = sizeof(cbuf); + + n = recvmsg(sock, &mh, MSG_DONTWAIT); + if (n < (ssize_t) sizeof(nxt_port_msg_t)) { + return -1; + } + + fd = -1; + + for (cm = CMSG_FIRSTHDR(&mh); cm != NULL; cm = CMSG_NXTHDR(&mh, cm)) { + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_RIGHTS) { + nxt_memcpy(&fd, CMSG_DATA(cm), sizeof(int)); + break; + } + } + + return fd; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index a41ae2696..8208896d6 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -262,6 +262,10 @@ main(int argc, char **argv) return 1; } + if (nxt_port_queued_fd_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CGROUP) if (nxt_cgroup_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 6061308bd..94fa6e331 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -87,6 +87,7 @@ nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_port_rpc_fd_test(nxt_thread_t *thr); +nxt_int_t nxt_port_queued_fd_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From 47735b4674fcc169cff8d495b5a1d689bbc81177 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 01:07:32 +0200 Subject: [PATCH 204/391] changelog: classify the queued-descriptor fix as Security The trigger is reachable from an application: nxt_router_get_port_handler() answers GET_PORT for any (pid, id) the message names, and #341 records that nothing authenticates the sender. A queued NEW_PORT reply that names a reused descriptor hands the peer a descriptor of the router's choosing, not of its own -- descriptor confusion, not a leak. Matches the existing entry for the port descriptor leak, which is Security on the same grounds. Co-Authored-By: Claude Opus 5 --- CHANGES | 2 +- docs/changes.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 2ee432521..25e09411c 100644 --- a/CHANGES +++ b/CHANGES @@ -24,7 +24,7 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 *) Change: upgrade contrib njs to 1.0.1. - *) Bugfix: a port message that was queued because the peer's socket was + *) Security: a port message that was queued because the peer's socket was not writable borrowed the descriptors it carried -- for NEW_PORT, another port's socket and queue -- so a port closed before the queue drained left the deferred sendmsg() naming a closed number, or an diff --git a/docs/changes.xml b/docs/changes.xml index dbda4e5cd..1e1d16c27 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -31,7 +31,7 @@ replaces anything that is not valid UTF-8 with U+FFFD before it serializes. - + a port message that was queued because the peer's socket was not writable borrowed the descriptors it carried -- for NEW_PORT, another port's socket From cd2e4a92dcc479315c12f2b95675e9d694bcd5f2 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 20:28:59 +0200 Subject: [PATCH 205/391] ci: build the core once, and let every leg unpack it Every matrix leg ran the same "Configure unit" and "Make unit" steps with the same flags. The same objects were compiled 26 times, about 50s a leg. Build them once in a new "core" job and upload build/ plus the generated root Makefile as an artifact. Each leg downloads it and builds only its own module. Nothing else changes: same flags, same modules, same tests, same pytest invocations. The artifact is a tarball, not a plain upload. actions/upload-artifact stores a zip and restores neither the executable bit nor mtimes, and the legs need both: they run build/unit_app_test, and `make ` decides what to rebuild from mtimes. The legs touch the unpacked tree to one timestamp. actions/checkout stamps the sources with the time it ran, which is later than the core job's build, so make would otherwise rebuild the core objects the artifact carries. Verified locally: without the touch `make php` recompiles nxt_unit.o, with it only nxt_php_sapi-php.o. The caches stay as they are. A leg still needs /opt/openssl-3.6 on disk at test time, because unitd's rpath points there, and it still needs the njs directory for the include paths recorded in build/autoconf.data. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-test.yml | 195 +++++++++++++++++++++++++------ 1 file changed, 161 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 996a02871..c5c333073 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -66,8 +66,145 @@ jobs: ' pkg/eol.json) echo "builds=${builds}" >> "$GITHUB_OUTPUT" + # The core build, once for the whole run. Every matrix leg below used to + # run the same "Configure unit" and "Make unit" steps with the same flags, + # so the same objects were compiled 26 times. This job does it once and + # ships the result as an artifact; the legs unpack it and build only their + # own module. Flags here are a copy of what the legs ran before -- keep + # the two in step if they ever change. + core: + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v5 + + - name: Read njs version + id: metadata + run: | + NJS_VERSION=$(sed -n "s/NJS_VERSION := \(.*\)/\1/p" pkg/contrib/src/njs/version) + echo "njs_version=${NJS_VERSION}" >> "$GITHUB_OUTPUT" + + - name: Install packages + run: sudo apt-get -y install libbrotli-dev ccache + + - name: Cache ccache + uses: actions/cache@v6 + with: + path: ~/.ccache + key: ccache-${{ runner.os }}-${{ runner.arch }}-core-${{ github.run_id }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-core- + + - name: Enable ccache + run: | + echo "/usr/lib/ccache" >> "$GITHUB_PATH" + echo "CCACHE_DIR=$HOME/.ccache" >> "$GITHUB_ENV" + echo "CCACHE_MAXSIZE=500M" >> "$GITHUB_ENV" + + - name: Cache OpenSSL 3.6 + id: cache-openssl36 + uses: actions/cache@v6 + with: + path: /opt/openssl-3.6 + key: openssl-${{ env.OPENSSL_VERSION }}-${{ runner.os }}-${{ runner.arch }} + + - name: Build OpenSSL 3.6 + if: steps.cache-openssl36.outputs.cache-hit != 'true' + run: | + sudo apt-get -y install build-essential + cd /tmp + wget -q "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" + tar -xzf "openssl-${OPENSSL_VERSION}.tar.gz" + cd "openssl-${OPENSSL_VERSION}" + ./Configure --prefix=/opt/openssl-3.6 shared no-docs + make -j$(nproc) + sudo mkdir -p /opt/openssl-3.6 + sudo make install_sw + + - name: Set OpenSSL 3.6 build environment + run: | + OSSL=/opt/openssl-3.6 + LIBDIR=$( [ -d "$OSSL/lib64" ] && echo "$OSSL/lib64" || echo "$OSSL/lib" ) + echo "OPENSSL36_INCDIR=${OSSL}/include" >> $GITHUB_ENV + echo "OPENSSL36_LIBDIR=${LIBDIR}" >> $GITHUB_ENV + + - name: Cache njs + id: cache-njs + uses: actions/cache@v6 + with: + path: njs + key: njs-freeunitorg-${{ steps.metadata.outputs.njs_version }}-${{ runner.os }}-${{ runner.arch }} + + - name: Clone njs repository + if: steps.cache-njs.outputs.cache-hit != 'true' + uses: actions/checkout@v5 + with: + repository: freeunitorg/njs + ref: '${{ steps.metadata.outputs.njs_version }}' + path: njs + + - name: Make njs + if: steps.cache-njs.outputs.cache-hit != 'true' + run: | + ./configure --no-libxml2 --no-zlib + make -j4 -k + working-directory: njs + + - name: Configure unit + # LD_LIBRARY_PATH is set only for this step so that auto/feature + # test binaries load OpenSSL 3.6 at runtime. It is NOT exported + # globally — doing so would break sudo (sudoers.so links against + # system libssl 3.3 which requires OPENSSL_3.3.0 from libcrypto, + # a symbol version that 3.6 does not export in the same slot). + env: + LD_LIBRARY_PATH: ${{ env.OPENSSL36_LIBDIR }} + run: | + ./configure \ + --tests \ + --openssl \ + --njs \ + --zlib \ + --zstd \ + --brotli \ + --otel \ + --cc-opt="-I${OPENSSL36_INCDIR} -I njs/src/ -I njs/build" \ + --ld-opt="-L${OPENSSL36_LIBDIR} -Wl,-rpath,${OPENSSL36_LIBDIR} -L njs/build" + + - name: Make unit + run: | + make -j$(nproc) -k || make + + # The default target is `all: unitd manpage`, so --tests above only + # makes this target available -- it does not build it. Without it + # build/unit_app_test does not exist and test_app_lifecycle.py's + # `libunit` parameter self-skips in every leg, silently: the runtime + # that needs no language toolchain and links libunit directly, i.e. + # the one that exercises the port/fd lifecycle with the least in the + # way, would contribute nothing while the legs still reported green. + - name: Make the libunit test application + run: make -j$(nproc) build/unit_app_test + + # tar, not a bare upload: actions/upload-artifact stores a zip and + # restores neither the executable bit nor mtimes. Both matter here -- + # the legs run build/unit_app_test directly, and `make ` decides + # what to rebuild from mtimes. The root Makefile is generated by + # configure (it just includes build/Makefile), so it travels too, and + # build/autoconf.data is what a module configure sources for the core's + # flags. + - name: Pack the core build + run: tar -czf /tmp/unit-core-build.tar.gz Makefile build + + - name: Upload the core build + uses: actions/upload-artifact@v4 + with: + name: unit-core-build + path: /tmp/unit-core-build.tar.gz + retention-days: 1 + compression-level: 0 + test: - needs: prepare + needs: [prepare, core] runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: @@ -285,39 +422,29 @@ jobs: ## Unit ## - - name: Configure unit - # LD_LIBRARY_PATH is set only for this step so that auto/feature - # test binaries load OpenSSL 3.6 at runtime. It is NOT exported - # globally — doing so would break sudo (sudoers.so links against - # system libssl 3.3 which requires OPENSSL_3.3.0 from libcrypto, - # a symbol version that 3.6 does not export in the same slot). - env: - LD_LIBRARY_PATH: ${{ env.OPENSSL36_LIBDIR }} - run: | - ./configure \ - --tests \ - --openssl \ - --njs \ - --zlib \ - --zstd \ - --brotli \ - --otel \ - --cc-opt="-I${OPENSSL36_INCDIR} -I njs/src/ -I njs/build" \ - --ld-opt="-L${OPENSSL36_LIBDIR} -Wl,-rpath,${OPENSSL36_LIBDIR} -L njs/build" - - - name: Make unit - run: | - make -j4 -k || make - - # The default target is `all: unitd manpage`, so --tests above only - # makes this target available -- it does not build it. Without it - # build/unit_app_test does not exist and test_app_lifecycle.py's - # `libunit` parameter self-skips in every leg, silently: the runtime - # that needs no language toolchain and links libunit directly, i.e. - # the one that exercises the port/fd lifecycle with the least in the - # way, would contribute nothing while the legs still reported green. - - name: Make the libunit test application - run: make -j4 build/unit_app_test + # The core build comes from the `core` job above: the same configure + # flags and the same objects, compiled once for the whole run instead + # of once per leg. Only this leg's own module is built below. + - name: Download the core build + uses: actions/download-artifact@v4 + with: + name: unit-core-build + path: /tmp + + - name: Unpack the core build + # actions/checkout stamps the sources with the time it ran, which is + # after the core job built them, so make would judge every core object + # out of date and rebuild the very work this job exists to reuse. + # Touching the whole build tree to one timestamp fixes that: make + # rebuilds on a *newer* prerequisite, and equal mtimes count as up to + # date. Everything under build/ came from this same commit, so there + # is no real staleness to mask. + run: | + tar -xzf /tmp/unit-core-build.tar.gz + find Makefile build -exec touch {} + + test -f build/autoconf.data + test -x build/sbin/unitd + test -x build/unit_app_test ## ## Go From 385244c9c2201e5fc7c4bf1166e26d9a6c9aff2e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 20:54:07 +0200 Subject: [PATCH 206/391] ci: run the Range x Content-Encoding tests with --zlib and --restart Three tests need a --zlib build and pytest --restart together, and no job supplied both, so they have skipped in every leg since they were written: test/test_static_compression.py test/test_static.py::test_static_range_gzip test/test_php_compression.py Give them their own job. Why a job rather than a flag on an existing leg, why --restart is what makes them safe while #167 is open, why the step greps its own output, why the PHP version comes from pkg/eol.json and why there is no sudo are all recorded in the job itself, where someone minded to remove one of them will read it. Measured: with --restart, 29 passed and 3 skipped in 10.95s, job 55s; without it the same command reports 32 skipped and the step now fails. The three skips are test_php_compression_small_body (#162) and large_random_body[br] and [zstd], which need brotli and zstd -- out of scope here, since #379 is about Range x Content-Encoding and gzip covers it. Closes #379. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-test.yml | 102 +++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c5c333073..40a06a61f 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -772,6 +772,108 @@ jobs: /usr/bin/pytest-3 --print-log test/test_capget_fallback.py if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} + # Range x Content-Encoding coverage (#379). test_static_compression.py, + # test_static.py::test_static_range_gzip and test_php_compression.py all + # skip "until #167 is fixed" unless option.restart is set. --restart is + # what makes running them safe while #167 stays open: each test gets a + # fresh unitd, so the stale pointer #167 describes + # (src/nxt_http_compression.c:86-89, freed at :1084) cannot reach across + # tests. This job does not touch nxt_http_compression.c. + # + # Neither existing pytest job has both flags this needs: the "unit"/"python" + # legs above build --zlib but never pass --restart, and --restart there + # would restart unitd for every test in the full "test" tree, not just + # these three files -- those legs are already the longest in this workflow. + # sanitize.yml's python leg passes --restart but configures without --zlib, + # and adding --zlib there would pull compression into every ASan/UBSan run + # for a leg that exists to red-line teardown, not codec coverage; its php + # leg configures without --restart on purpose (the teardown rmtree trips + # over test_php_application_forbidden's restricted-permission fixture). + # A small dedicated job avoids all of that: a minimal build (--zlib plus + # the php module, nothing else) running only these three files with + # --restart, so the added cost is three fresh-unitd tests, not a slower + # existing leg. + compression-range-tests: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v5 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libssl-dev libpcre2-dev \ + zlib1g-dev + + # Take the version from pkg/eol.json instead of pinning "8": setup-php + # resolves a bare "8" to whatever is newest that day, so the job would + # drift without a commit and an old run would not reproduce. Any + # supported version exercises Unit's compression the same way. + - name: Pick the PHP version + id: php + run: | + version="$(jq -r '.runtimes.php[0].version' pkg/eol.json)" + + case "${version}" in + ''|null|*[!0-9.]*) + echo "::error::no php runtime in pkg/eol.json" >&2 + exit 1 + ;; + esac + + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - uses: shivammathur/setup-php@v2 + with: + php-version: '${{ steps.php.outputs.version }}' + extensions: none + env: + update: true + + - name: Configure unit + run: ./configure --openssl --zlib + + - name: Make unit + run: make -j$(nproc) + + - name: Configure php + run: ./configure php + + - name: Make php + run: make -j$(nproc) php + + - uses: actions/setup-python@v7 + with: + python-version: '3' + + - name: Install pytest + run: sudo apt install -y python3-pytest + + # TZ pin: same reasoning as the "unit"/"python" leg above. + # + # The greps are the point: a run that skips all three files also exits + # 0, which is the #379 failure itself, not a hypothetical. Assert a + # PASSED line per file, as the OpenSSL 4.0 leg asserts on + # nxt_auto_config.h. pipefail keeps pytest's exit status once tee is + # in the pipeline. + # No sudo here, unlike the language legs: nothing in these three files + # needs privilege, and running as the job user keeps the temp tree out + # of root's ownership. Deliberate, not an oversight. + - name: Run Range x Content-Encoding tests + env: + TZ: Asia/Kolkata + run: | + set -o pipefail + pytest-3 --print-log --restart \ + test/test_static_compression.py \ + "test/test_static.py::test_static_range_gzip" \ + test/test_php_compression.py | tee pytest.out + + grep -qE 'test_static_compression\.py::.* PASSED' pytest.out + grep -q 'test_static.py::test_static_range_gzip PASSED' pytest.out + grep -qE 'test_php_compression\.py::.* PASSED' pytest.out + # C unit-test suite (src/test/ -> build/tests): HTTP parser, port/IPC # fault-injection, lvlhsh, mp, rbtree, base64, utf8, clone creds, etc. # Previously compiled but never executed in CI; runs here on every PR as its From e58664348bac2979cc36941319dfe91554372434 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 21:21:29 +0200 Subject: [PATCH 207/391] http: hold the compression state per configuration The compression state lived in four process globals, all allocated from the router configuration that parsed them: the Accept-Encoding query, the MIME types rule, the enabled compressor table and its length. Nothing refilled or cleared them when that configuration was freed, so a configuration with a "compression" block followed by one without left every one of them pointing into a freed pool, and the next request killed the router. Hold them in an nxt_http_comp_conf_t hanging off nxt_router_conf_t instead, and reach it from the request as r->conf->socket_conf->router_conf. A request therefore reads the configuration it was routed under, and a configuration that is gone takes its compression state with it. The struct is published only once it is fully built, so a request that arrives mid-assembly sees no compression rather than a half-filled table. The per-response context now also keeps the chosen compressor's type entry rather than an index into that table. The type table is static, so a body that is still being compressed when the configuration is replaced finishes against memory that outlives it. With that, the three test files that could only run under --restart run anywhere, so drop their skip guards. Without the fix, running them in one unitd fails: test_static.py reports "same pid router" as the router dies and restarts under it. Closes: #167 Co-Authored-By: Claude Opus 5 (1M context) --- src/nxt_http_compression.c | 157 +++++++++++++++++++++----------- src/nxt_router.c | 18 +++- src/nxt_router.h | 10 ++ test/test_php_compression.py | 17 ---- test/test_static.py | 8 +- test/test_static_compression.py | 43 ++++++--- 6 files changed, 158 insertions(+), 95 deletions(-) diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 970979f10..d5a51b82a 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -69,6 +69,15 @@ struct nxt_http_comp_compressor_s { struct nxt_http_comp_ctx_s { nxt_uint_t idx; + /* + * The compressor's type table entry, copied out of the configuration + * when the choice is applied. The table is static and lives as long as + * the process, so a body that is still being compressed when the + * configuration is replaced keeps working from here instead of indexing + * a per-configuration array that may already be freed. + */ + const nxt_http_comp_type_t *type; + /* * The compressor nxt_http_comp_check_acceptable() chose, or -1 when the * request will not be compressed. Kept apart from "idx", which is only @@ -83,10 +92,19 @@ struct nxt_http_comp_ctx_s { }; -static nxt_tstr_t *nxt_http_comp_accept_encoding_query; -static nxt_http_route_rule_t *nxt_http_comp_mime_types_rule; -static nxt_http_comp_compressor_t *nxt_http_comp_enabled_compressors; -static nxt_uint_t nxt_http_comp_nr_enabled_compressors; +/* + * Everything here is allocated from the router configuration's pools, so it + * is held per configuration and reached through the request. It used to be + * four process-global pointers, which dangled as soon as the configuration + * they came from was freed -- reconfiguring compression away killed the + * router on the next request (#167). + */ +struct nxt_http_comp_conf_s { + nxt_tstr_t *accept_encoding_query; + nxt_http_route_rule_t *mime_types_rule; + nxt_http_comp_compressor_t *enabled; + nxt_uint_t nr_enabled; +}; static nxt_thread_declare_data(nxt_http_comp_ctx_t, nxt_http_comp_compressor_ctx); @@ -148,16 +166,21 @@ static const nxt_http_comp_type_t nxt_http_comp_compressors[] = { }; +nxt_inline nxt_http_comp_conf_t * +nxt_http_comp_request_conf(const nxt_http_request_t *r) +{ + return r->conf->socket_conf->router_conf->compression; +} + + static ssize_t nxt_http_comp_compress(uint8_t *dst, size_t dst_size, const uint8_t *src, size_t src_size, bool last) { nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - nxt_http_comp_compressor_t *compressor; const nxt_http_comp_operations_t *cops; - compressor = &nxt_http_comp_enabled_compressors[ctx->idx]; - cops = compressor->type->cops; + cops = ctx->type->cops; return cops->deflate(&ctx->ctx, src, src_size, dst, dst_size, last); } @@ -167,11 +190,9 @@ static size_t nxt_http_comp_bound(size_t size) { nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - nxt_http_comp_compressor_t *compressor; const nxt_http_comp_operations_t *cops; - compressor = &nxt_http_comp_enabled_compressors[ctx->idx]; - cops = compressor->type->cops; + cops = ctx->type->cops; return cops->bound(&ctx->ctx, size); } @@ -419,16 +440,15 @@ nxt_http_comp_wants_compression(void) static nxt_uint_t -nxt_http_comp_compressor_lookup_enabled(const nxt_str_t *token) +nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, + const nxt_str_t *token) { if (token->start[0] == '*') { return NXT_HTTP_COMP_SCHEME_IDENTITY; } - for (nxt_uint_t i = 0; i < nxt_http_comp_nr_enabled_compressors; i++) { - if (nxt_strstr_eq(token, - &nxt_http_comp_enabled_compressors[i].type->token)) - { + for (nxt_uint_t i = 0; i < conf->nr_enabled; i++) { + if (nxt_strstr_eq(token, &conf->enabled[i].type->token)) { return i; } } @@ -454,7 +474,8 @@ nxt_http_comp_compressor_lookup_enabled(const nxt_str_t *token) * 'identity;q=0' seems to basically mean the same thing... */ static nxt_int_t -nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) +nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, + nxt_http_request_t *r, const nxt_str_t *token) { bool identity_allowed = true; char *str, *tkn, *tail, *cur; @@ -501,12 +522,12 @@ nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) enc.start = (u_char *)tkn; enc.length = qptr != NULL ? (size_t)(qptr - tkn) : strlen(tkn); - ecidx = nxt_http_comp_compressor_lookup_enabled(&enc); + ecidx = nxt_http_comp_compressor_lookup_enabled(conf, &enc); if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) { continue; } - scheme = nxt_http_comp_enabled_compressors[ecidx].type->scheme; + scheme = conf->enabled[ecidx].type->scheme; if (qval == 0.0 && scheme == NXT_HTTP_COMP_SCHEME_IDENTITY) { identity_allowed = false; @@ -529,7 +550,8 @@ nxt_http_comp_select_compressor(nxt_http_request_t *r, const nxt_str_t *token) static nxt_int_t -nxt_http_comp_set_header(nxt_http_request_t *r, nxt_uint_t comp_idx) +nxt_http_comp_set_header(const nxt_http_comp_conf_t *conf, + nxt_http_request_t *r, nxt_uint_t comp_idx) { const nxt_str_t *token; nxt_http_field_t *f; @@ -542,7 +564,7 @@ nxt_http_comp_set_header(nxt_http_request_t *r, nxt_uint_t comp_idx) return NXT_ERROR; } - token = &nxt_http_comp_enabled_compressors[comp_idx].type->token; + token = &conf->enabled[comp_idx].type->token; *f = (nxt_http_field_t){}; @@ -806,14 +828,17 @@ nxt_http_comp_merge_vary(nxt_http_request_t *r) nxt_int_t nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) { - nxt_int_t ret, idx; - nxt_str_t accept_encoding, mime_type = {}; - nxt_router_conf_t *rtcf; - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_int_t ret, idx; + nxt_str_t accept_encoding, mime_type = {}; + nxt_router_conf_t *rtcf; + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_conf_t *conf = nxt_http_comp_request_conf(r); *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1, .sel_idx = -1 }; - if (nxt_http_comp_nr_enabled_compressors == 0) { + /* A built configuration always holds identity, so NULL is the only + "no compression" state. */ + if (conf == NULL) { return NXT_OK; } @@ -836,8 +861,8 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_OK; } - if (nxt_http_comp_mime_types_rule != NULL) { - ret = nxt_http_route_test_rule(r, nxt_http_comp_mime_types_rule, + if (conf->mime_types_rule != NULL) { + ret = nxt_http_route_test_rule(r, conf->mime_types_rule, mime_type.start, mime_type.length); if (ret == 0) { @@ -868,13 +893,13 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_ERROR; } - ret = nxt_tstr_query(task, r->tstr_query, - nxt_http_comp_accept_encoding_query, &accept_encoding); + ret = nxt_tstr_query(task, r->tstr_query, conf->accept_encoding_query, + &accept_encoding); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; } - idx = nxt_http_comp_select_compressor(r, &accept_encoding); + idx = nxt_http_comp_select_compressor(conf, r, &accept_encoding); if (idx == -1) { return NXT_HTTP_NOT_ACCEPTABLE; } @@ -964,6 +989,7 @@ nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) nxt_int_t idx; nxt_off_t min_len; nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_conf_t *conf; nxt_http_comp_compressor_t *compressor; idx = ctx->sel_idx; @@ -972,7 +998,12 @@ nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) return NXT_OK; } - compressor = &nxt_http_comp_enabled_compressors[idx]; + /* + * Reached only when nxt_http_comp_check_acceptable() chose a compressor, + * which it does only from a non-NULL configuration. + */ + conf = nxt_http_comp_request_conf(r); + compressor = &conf->enabled[idx]; if (r->resp.content_length_n > -1) { ctx->resp_clen = r->resp.content_length_n; @@ -987,14 +1018,15 @@ nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) return NXT_OK; } - nxt_http_comp_set_header(r, idx); + nxt_http_comp_set_header(conf, r, idx); if (nxt_slow_path(nxt_http_comp_weaken_etag(r) != NXT_OK)) { return NXT_ERROR; } ctx->idx = idx; - ctx->ctx.level = nxt_http_comp_enabled_compressors[idx].opts.level; + ctx->type = compressor->type; + ctx->ctx.level = compressor->opts.level; err = compressor->type->cops->init(&ctx->ctx); if (nxt_slow_path(err)) { @@ -1034,6 +1066,7 @@ nxt_http_comp_compressor_is_valid(const nxt_str_t *token) static nxt_int_t nxt_http_comp_set_compressor(nxt_task_t *task, nxt_router_conf_t *rtcf, + nxt_http_comp_conf_t *conf, const nxt_conf_value_t *comp, nxt_uint_t index) { nxt_int_t ret; @@ -1052,7 +1085,7 @@ nxt_http_comp_set_compressor(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_get_string(obj, &token); cidx = nxt_http_comp_compressor_token2idx(&token); - compr = &nxt_http_comp_enabled_compressors[index]; + compr = &conf->enabled[index]; compr->type = &nxt_http_comp_compressors[cidx]; compr->opts.level = compr->type->def_compr; @@ -1084,29 +1117,35 @@ nxt_int_t nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, const nxt_conf_value_t *comp_conf) { - nxt_int_t ret; - nxt_uint_t n = 1; /* 'identity' */ - nxt_conf_value_t *comps, *mimes; + nxt_int_t ret; + nxt_uint_t n = 1; /* 'identity' */ + nxt_conf_value_t *comps, *mimes; + nxt_http_comp_conf_t *conf; static const nxt_str_t accept_enc_str = nxt_string("$header_accept_encoding"); static const nxt_str_t comps_str = nxt_string("compressors"); static const nxt_str_t mimes_str = nxt_string("types"); + conf = nxt_mp_zalloc(rtcf->mem_pool, sizeof(nxt_http_comp_conf_t)); + if (nxt_slow_path(conf == NULL)) { + return NXT_ERROR; + } + mimes = nxt_conf_get_object_member(comp_conf, &mimes_str, NULL); if (mimes != NULL) { - nxt_http_comp_mime_types_rule = + conf->mime_types_rule = nxt_http_route_types_rule_create(task, rtcf->mem_pool, mimes); - if (nxt_slow_path(nxt_http_comp_mime_types_rule == NULL)) { + if (nxt_slow_path(conf->mime_types_rule == NULL)) { return NXT_ERROR; } } - nxt_http_comp_accept_encoding_query = + conf->accept_encoding_query = nxt_tstr_compile(rtcf->tstr_state, &accept_enc_str, NXT_TSTR_STRZ); - if (nxt_slow_path(nxt_http_comp_accept_encoding_query == NULL)) { + if (nxt_slow_path(conf->accept_encoding_query == NULL)) { return NXT_ERROR; } @@ -1120,30 +1159,40 @@ nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, } else { n += nxt_conf_object_members_count(comps); } - nxt_http_comp_nr_enabled_compressors = n; + conf->nr_enabled = n; - nxt_http_comp_enabled_compressors = - nxt_mp_zalloc(rtcf->mem_pool, - sizeof(nxt_http_comp_compressor_t) * n); + conf->enabled = nxt_mp_zalloc(rtcf->mem_pool, + sizeof(nxt_http_comp_compressor_t) * n); + if (nxt_slow_path(conf->enabled == NULL)) { + return NXT_ERROR; + } - nxt_http_comp_enabled_compressors[0] = + conf->enabled[0] = (nxt_http_comp_compressor_t){ .type = &nxt_http_comp_compressors[0], .opts.level = NXT_COMP_LEVEL_UNSET, .opts.min_len = -1 }; if (nxt_conf_type(comps) == NXT_CONF_OBJECT) { - return nxt_http_comp_set_compressor(task, rtcf, comps, 1); - } + ret = nxt_http_comp_set_compressor(task, rtcf, conf, comps, 1); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } - for (nxt_uint_t i = 1; i < nxt_http_comp_nr_enabled_compressors; i++) { - nxt_conf_value_t *obj; + } else { + for (nxt_uint_t i = 1; i < conf->nr_enabled; i++) { + nxt_conf_value_t *obj; - obj = nxt_conf_get_array_element(comps, i - 1); - ret = nxt_http_comp_set_compressor(task, rtcf, obj, i); - if (ret == NXT_ERROR) { - return NXT_ERROR; + obj = nxt_conf_get_array_element(comps, i - 1); + ret = nxt_http_comp_set_compressor(task, rtcf, conf, obj, i); + if (ret == NXT_ERROR) { + return NXT_ERROR; + } } } + /* One publish point: a failure above leaves rtcf->compression NULL. */ + + rtcf->compression = conf; + return NXT_OK; } diff --git a/src/nxt_router.c b/src/nxt_router.c index 1f242c0a7..70ec73c0b 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -2506,6 +2506,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, *otel_batching, *otel_proto; #endif nxt_conf_value_t *root, *conf, *http, *value, *websocket; + nxt_conf_value_t *comp; nxt_conf_value_t *applications, *application, *settings; nxt_conf_value_t *listeners, *listener; nxt_socket_conf_t *skcf; @@ -2848,6 +2849,18 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, websocket = nxt_conf_get_path(root, &websocket_path); + /* + * Compression is configured once for the whole router, not per listener. + * The return is deliberately not checked: a "compression" block the + * validator accepts but this cannot build -- one with no "compressors", + * which is not a required member -- leaves compression off rather than + * rejecting the configuration, which is what it has always done. + */ + comp = nxt_conf_get_path(root, &compression_path); + if (comp != NULL) { + (void) nxt_http_comp_compression_init(task, rtcf, comp); + } + listeners = nxt_conf_get_path(root, &listeners_path); if (listeners != NULL) { @@ -2906,7 +2919,6 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_str_null(&skcf->body_temp_path); if (http != NULL) { - nxt_conf_value_t *comp; ret = nxt_conf_map_object(mp, http, nxt_router_http_conf, nxt_nitems(nxt_router_http_conf), @@ -2916,10 +2928,6 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, goto fail; } - comp = nxt_conf_get_path(root, &compression_path); - if (comp != NULL) { - nxt_http_comp_compression_init(task, rtcf, comp); - } } if (websocket != NULL) { diff --git a/src/nxt_router.h b/src/nxt_router.h index 9f1844edd..2bff5332d 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -23,6 +23,7 @@ typedef struct nxt_upstream_s nxt_upstream_t; typedef struct nxt_upstreams_s nxt_upstreams_t; typedef struct nxt_router_access_log_s nxt_router_access_log_t; typedef struct nxt_router_access_log_format_s nxt_router_access_log_format_t; +typedef struct nxt_http_comp_conf_s nxt_http_comp_conf_t; #define NXT_HTTP_ACTION_ERROR ((nxt_http_action_t *) -1) @@ -56,6 +57,15 @@ typedef struct { nxt_tstr_cond_t log_cond; nxt_router_access_log_t *access_log; nxt_router_access_log_format_t *log_format; + + /* + * Compression state, allocated from mem_pool above and reachable from a + * request as r->conf->socket_conf->router_conf->compression. NULL when + * this configuration has no "compression" block. Per-configuration + * rather than process-global so that reconfiguring compression away + * cannot leave a pointer into a freed pool behind (#167). + */ + nxt_http_comp_conf_t *compression; } nxt_router_conf_t; diff --git a/test/test_php_compression.py b/test/test_php_compression.py index c23a59dc8..3555a5785 100644 --- a/test/test_php_compression.py +++ b/test/test_php_compression.py @@ -5,29 +5,12 @@ import pytest from unit.applications.lang.php import ApplicationPHP -from unit.option import option prerequisites = {'modules': {'php': 'any'}} client = ApplicationPHP() -@pytest.fixture(autouse=True) -def requires_restart_mode(): - """ - Configuring compression is not reversible within one unitd: the module - keeps its state in process globals pointing into the router configuration - that was current when they were parsed, and a later configuration without - a compression block neither reinitialises nor clears them, so the next - request dereferences a freed pool and the router dies (#167). - - Without --restart the whole session shares one unitd, so these tests would - crash whichever test runs next. With it, each test gets its own unitd and - the state cannot escape. Drop this fixture once #167 is fixed. - """ - if not option.restart: - pytest.skip('needs --restart until #167 is fixed') - def configure_compression(encoding='gzip'): conf = { "compression": { diff --git a/test/test_static.py b/test/test_static.py index c681b0bda..dd584b584 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -1717,15 +1717,9 @@ def test_static_range_file_changes_between_requests(temp_dir): def test_static_range_gzip(temp_dir): # Compression must be skipped for a 206 (a coded slice would be - # meaningless) but still work for the plain 200 alongside it. The - # module keeps global state, so this needs a fresh unitd (#167). + # meaningless) but still work for the plain 200 alongside it. import gzip - from unit.option import option - - if not option.restart: - pytest.skip('needs --restart until #167 is fixed') - resp = client.conf( { "http": { diff --git a/test/test_static_compression.py b/test/test_static_compression.py index e0356f864..f143bf048 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -3,22 +3,10 @@ import pytest from unit.applications.proto import ApplicationProto -from unit.option import option client = ApplicationProto() -@pytest.fixture(autouse=True) -def requires_restart_mode(): - """ - Configuring compression is not reversible within one unitd (#167), so - these tests need their own unitd or they crash whichever test runs next. - Same reasoning as test_php_compression.py; drop both once #167 is fixed. - """ - if not option.restart: - pytest.skip('needs --restart until #167 is fixed') - - @pytest.fixture(autouse=True) def setup_method_fixture(temp_dir): assets_dir = f'{temp_dir}/assets' @@ -58,6 +46,37 @@ def test_static_compression_baseline(): assert resp['headers']['Content-Encoding'] == 'gzip', 'gzip applied' +def test_static_compression_removed_between_requests(): + # #167: the compression state used to be process-global and allocated + # from the router configuration that parsed it, so a configuration + # without a "compression" block left it pointing into a freed pool and + # the next request killed the router. Pin the transition itself: every + # other test in this file only ever configures compression, so the bug + # reproduced through the file order of a whole suite run rather than + # through any one file. + headers = { + 'Host': 'localhost', + 'Accept-Encoding': 'gzip', + 'Connection': 'close', + } + + resp = client.get(url='/big.css', headers=headers) + assert resp['headers']['Content-Encoding'] == 'gzip', 'gzip before' + + assert 'success' in client.conf_delete( + 'settings/http/compression' + ), 'compression removed' + + resp = client.get(url='/big.css', headers=headers) + assert resp['status'] == 200, 'identity after the block is removed' + assert 'Content-Encoding' not in resp['headers'], 'no coding after' + + # The second request is the one that used to find a freed pool: the + # first may be answered before the old configuration is released. + resp = client.get(url='/big.css', headers=headers) + assert resp['status'] == 200, 'router still serving' + + def test_static_compression_precondition_does_not_mask_406(): # RFC 9110 Sect. 13.2.1: an ordinary failure outranks a precondition. A # client that refuses every encoding cannot be served at all, so the From 4a32bc8245618015a0088175707497915a324fe4 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 21:21:29 +0200 Subject: [PATCH 208/391] changelog: compression state is held per configuration Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 10 ++++++++++ docs/changes.xml | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGES b/CHANGES index 25e09411c..fa5637d27 100644 --- a/CHANGES +++ b/CHANGES @@ -31,6 +31,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 unrelated descriptor that had since reused it. The queued copy now owns duplicates and closes them after the send (#388). + *) Bugfix: the router died on the first request after a configuration + that had a "compression" block was replaced by one that had none. + The compression state was held in process globals allocated from the + router configuration that parsed them, and a later configuration + neither refilled nor cleared them, so the next request read a freed + pool. The state is now held per configuration and reached through + the request, and the compressor a response is already using is kept + with that response, so a body still being compressed when the + configuration is replaced also finishes (#167). + *) Bugfix: a static response that was subject to content negotiation now carries "Vary: Accept-Encoding". Without it a shared cache could store whichever representation it saw first and serve it to every client -- diff --git a/docs/changes.xml b/docs/changes.xml index 1e1d16c27..351cc21ac 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -83,6 +83,19 @@ representation, per RFC 9110. + + +the router died on the first request after a configuration that had a +"compression" block was replaced by one that had none. The compression +state was held in process globals allocated from the router configuration +that parsed them, and a later configuration neither refilled nor cleared +them, so the next request read a freed pool. The state is now held per +configuration and reached through the request, and the compressor a +response is already using is kept with that response, so a body still +being compressed when the configuration is replaced also finishes (#167). + + + a static response that was subject to content negotiation now carries From df9843b0ff434ff3dcc53ee015febd604f13306f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 21:26:26 +0200 Subject: [PATCH 209/391] ci: drop the dedicated Range x Content-Encoding job The job existed because test_static_compression.py, test_php_compression.py and test_static.py::test_static_range_gzip skipped themselves unless --restart was set, and no existing leg passed both --restart and --zlib. The skip guards are gone, so the ordinary legs cover all three: the "unit" leg runs the whole test tree, one python shard takes each shared file, and "test/test_php*" reaches test_php_compression.py on every php leg. Each of those legs builds --zlib through the shared core. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-test.yml | 102 ------------------------------- 1 file changed, 102 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 40a06a61f..c5c333073 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -772,108 +772,6 @@ jobs: /usr/bin/pytest-3 --print-log test/test_capget_fallback.py if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} - # Range x Content-Encoding coverage (#379). test_static_compression.py, - # test_static.py::test_static_range_gzip and test_php_compression.py all - # skip "until #167 is fixed" unless option.restart is set. --restart is - # what makes running them safe while #167 stays open: each test gets a - # fresh unitd, so the stale pointer #167 describes - # (src/nxt_http_compression.c:86-89, freed at :1084) cannot reach across - # tests. This job does not touch nxt_http_compression.c. - # - # Neither existing pytest job has both flags this needs: the "unit"/"python" - # legs above build --zlib but never pass --restart, and --restart there - # would restart unitd for every test in the full "test" tree, not just - # these three files -- those legs are already the longest in this workflow. - # sanitize.yml's python leg passes --restart but configures without --zlib, - # and adding --zlib there would pull compression into every ASan/UBSan run - # for a leg that exists to red-line teardown, not codec coverage; its php - # leg configures without --restart on purpose (the teardown rmtree trips - # over test_php_application_forbidden's restricted-permission fixture). - # A small dedicated job avoids all of that: a minimal build (--zlib plus - # the php module, nothing else) running only these three files with - # --restart, so the added cost is three fresh-unitd tests, not a slower - # existing leg. - compression-range-tests: - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v5 - - - name: Install packages - run: | - sudo apt-get -y update - sudo apt-get -y install build-essential libssl-dev libpcre2-dev \ - zlib1g-dev - - # Take the version from pkg/eol.json instead of pinning "8": setup-php - # resolves a bare "8" to whatever is newest that day, so the job would - # drift without a commit and an old run would not reproduce. Any - # supported version exercises Unit's compression the same way. - - name: Pick the PHP version - id: php - run: | - version="$(jq -r '.runtimes.php[0].version' pkg/eol.json)" - - case "${version}" in - ''|null|*[!0-9.]*) - echo "::error::no php runtime in pkg/eol.json" >&2 - exit 1 - ;; - esac - - echo "version=${version}" >> "$GITHUB_OUTPUT" - - - uses: shivammathur/setup-php@v2 - with: - php-version: '${{ steps.php.outputs.version }}' - extensions: none - env: - update: true - - - name: Configure unit - run: ./configure --openssl --zlib - - - name: Make unit - run: make -j$(nproc) - - - name: Configure php - run: ./configure php - - - name: Make php - run: make -j$(nproc) php - - - uses: actions/setup-python@v7 - with: - python-version: '3' - - - name: Install pytest - run: sudo apt install -y python3-pytest - - # TZ pin: same reasoning as the "unit"/"python" leg above. - # - # The greps are the point: a run that skips all three files also exits - # 0, which is the #379 failure itself, not a hypothetical. Assert a - # PASSED line per file, as the OpenSSL 4.0 leg asserts on - # nxt_auto_config.h. pipefail keeps pytest's exit status once tee is - # in the pipeline. - # No sudo here, unlike the language legs: nothing in these three files - # needs privilege, and running as the job user keeps the temp tree out - # of root's ownership. Deliberate, not an oversight. - - name: Run Range x Content-Encoding tests - env: - TZ: Asia/Kolkata - run: | - set -o pipefail - pytest-3 --print-log --restart \ - test/test_static_compression.py \ - "test/test_static.py::test_static_range_gzip" \ - test/test_php_compression.py | tee pytest.out - - grep -qE 'test_static_compression\.py::.* PASSED' pytest.out - grep -q 'test_static.py::test_static_range_gzip PASSED' pytest.out - grep -qE 'test_php_compression\.py::.* PASSED' pytest.out - # C unit-test suite (src/test/ -> build/tests): HTTP parser, port/IPC # fault-injection, lvlhsh, mp, rbtree, base64, utf8, clone creds, etc. # Previously compiled but never executed in CI; runs here on every PR as its From 362f584f2c35c0c3771d70de999b24fd85787f72 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 16 Sep 2026 23:31:43 +0200 Subject: [PATCH 210/391] websocket: validate text frames and close reasons as UTF-8 A text message and the reason of a close frame are UTF-8 by definition (RFC 6455 Sections 5.6 and 5.5.1). Nothing looked at either: the payload of a text frame was forwarded unread, and the close frame only had its 2-byte code checked. A client could send anything and never see the 1007 the protocol asks for when the data cannot be interpreted (Section 8.1, Section 7.4.1) -- the constant for it was the one code in the table nothing sent. The router is the endpoint that fails the connection, so the check goes where a fully buffered frame is handled and nothing in libunit or in the language modules changes. The frame is read as it arrived, client mask included: every byte is XORed with mask[i % 4] as it is walked, the way the existing close-code check reads its two bytes. Binary payloads are not text and are not read. A sequence the end of a frame cuts in half is not an error yet (Section 5.6: "a particular text frame might include a partial UTF-8 sequence; however, the whole message MUST contain valid UTF-8"). The at most three trailing bytes are kept in the connection state and completed by the frame that continues the message; the state is dropped when a new message starts, on close, and with the rest of the connection state on keepalive (nxt_h1proto.c, the memzero before conn) and connection teardown. A text message that ends with a partial sequence is invalid, and so is a close reason that does. nxt_utf8_decode() cannot say where a bad byte was: it answers 0xFFFFFFFF for "invalid" and for "truncated at the end of the buffer" alike, and leaves *start where it was. The two are told apart here -- a partial sequence is completed with the smallest continuation bytes a valid encoding could have and handed to the decoder, so the prefix test cannot drift from it. The decoder already rejects surrogates and overlong forms (src/nxt_utf8.c); that is not repeated, and nxt_utf8.c is untouched. A frame whose buffer chain runs out before its payload does answers 1002 "Frame shorter than its header" instead of 1007, so a reader fault is not reported as an encoding one; the branch cannot be taken while every caller hands over a frame that is buffered whole. The walker's one remaining NXT_ERROR says at the line why it cannot be taken either. An RSV bit belongs to an extension (RFC 6455 Section 5.2), and RFC 7692 Section 6 puts the "Per-Message Compressed" bit, RSV1, on the first fragment of a compressed data message. Its payload is not UTF-8 until the application decodes it, so a message whose first frame carries an RSV bit is not text here, and neither is a message with one on a continuation frame. The close reason is still read with an RSV bit set: permessage-deflate and its relatives are data-message extensions, and the 2-byte code above it is read the same way. Nothing else about RSV handling changes -- the router still forwards such frames, which is the gap the skipped 3_1..3_7 cases describe, so the guarantee this adds is conditional. Recording whether the 101 response carried Sec-WebSocket-Extensions, rejecting un-negotiated RSV bits with 1002, and narrowing this exemption is its own change and is not in this commit. Tests: the six commented Autobahn cases are enabled -- 6_3_1 and 6_3_2 in test_asgi_websockets.py, test_java_websockets.py and test_node_websockets.py, and 7_5_1 in the same three files -- and the three notes that said this was the application's job are corrected. The payloads are written as bytes: as str the frame writer encodes them to valid UTF-8, so the case would not have exercised the validator. Each 6_3_1 opens its own connection, because the one the case was commented out next to is closed by close_connection() above it. The "# 6_4_1 ... 6_4_4 FAIL" label is gone: it had no body in any suite, and the fail-fast cases it names would need the payload to be validated as it arrives, which this reader -- it buffers a whole frame before dispatch -- does not do. Two cases are added to the ASGI suite for the ends of the validator, since the code is the same for all three legs: a text message whose last frame ends inside a multi-byte sequence (a lone 0xCE) is invalid, and a close reason that is valid UTF-8 but not ASCII (CE BA) is accepted. The binary exemption is already covered by 1_2_2..1_2_8, which send b'\xfe' * N as OP_BINARY and assert the echoed frame. The three statics join the file's forward-declaration block; the skip test in the walker is the cold path for a text frame and is marked so; the payload length is cast where it narrows from uint64_t to size_t; and the six double blank lines the uncommented blocks left inside their function bodies are collapsed to one, the only six in test/. Co-Authored-By: DeepSeek V4.1 Flash Harness: DeepSeek Harness --- src/nxt_h1proto.h | 14 ++ src/nxt_h1proto_websocket.c | 270 ++++++++++++++++++++++++++++++++++- test/test_asgi_websockets.py | 82 ++++++----- test/test_java_websockets.py | 64 ++++----- test/test_node_websockets.py | 64 ++++----- 5 files changed, 394 insertions(+), 100 deletions(-) diff --git a/src/nxt_h1proto.h b/src/nxt_h1proto.h index b324db8d1..bff3dd2c8 100644 --- a/src/nxt_h1proto.h +++ b/src/nxt_h1proto.h @@ -16,6 +16,17 @@ typedef struct nxt_h1p_websocket_timer_s nxt_h1p_websocket_timer_t; +/* + * The trailing bytes of a UTF-8 sequence the end of a frame cut in half. + * A sequence is at most four bytes, so three of them can be incomplete; + * they are completed by the frame that continues the message. + */ +typedef struct { + uint8_t len; + u_char tail[3]; +} nxt_h1p_ws_utf8_t; + + struct nxt_h1proto_s { nxt_http_request_parse_t parser; nxt_http_chunk_parse_t chunked_parse; @@ -34,6 +45,9 @@ struct nxt_h1proto_s { uint8_t websocket_cont_expected; /* 1 bit */ uint8_t websocket_closed; /* 1 bit */ + uint8_t websocket_text; /* 1 bit */ + + nxt_h1p_ws_utf8_t websocket_utf8; uint32_t header_size; diff --git a/src/nxt_h1proto_websocket.c b/src/nxt_h1proto_websocket.c index d2cbee7ce..2f5a69192 100644 --- a/src/nxt_h1proto_websocket.c +++ b/src/nxt_h1proto_websocket.c @@ -34,6 +34,11 @@ static void hxt_h1p_send_ws_error(nxt_task_t *task, nxt_http_request_t *r, const nxt_ws_error_t *err, ...); static void nxt_h1p_conn_ws_error_sent(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data); +static size_t nxt_h1p_ws_utf8_seqlen(u_char c); +static nxt_int_t nxt_h1p_ws_utf8_begin(const u_char *seq, size_t n); +static nxt_int_t nxt_h1p_ws_utf8_validate(nxt_h1p_ws_utf8_t *state, + nxt_buf_t *b, u_char *start, size_t skip, size_t length, + const u_char *mask, nxt_uint_t final); static const nxt_conn_state_t nxt_h1p_read_ws_frame_header_state; static const nxt_conn_state_t nxt_h1p_read_ws_frame_payload_state; @@ -71,6 +76,12 @@ static const nxt_ws_error_t nxt_ws_err_cont_expected = { static const nxt_ws_error_t nxt_ws_err_invalid_length = { NXT_WEBSOCKET_CR_PROTOCOL_ERROR, 0, nxt_string("Invalid extended payload length") }; +static const nxt_ws_error_t nxt_ws_err_invalid_utf8 = { + NXT_WEBSOCKET_CR_INVALID_DATA, + 0, nxt_string("Invalid UTF-8") }; +static const nxt_ws_error_t nxt_ws_err_truncated = { + NXT_WEBSOCKET_CR_PROTOCOL_ERROR, + 0, nxt_string("Frame shorter than its header") }; void nxt_h1p_websocket_first_frame_start(nxt_task_t *task, nxt_http_request_t *r, @@ -326,6 +337,21 @@ nxt_h1p_conn_ws_frame_header_read(nxt_task_t *task, void *obj, void *data) wsh->opcode); return; } + + /* + * A message starts here: remember whether its payload is text, + * and drop a sequence the previous message left incomplete. A + * frame with an RSV bit set belongs to an extension this router + * does not implement (RFC 6455 Section 5.2; RFC 7692 Section 6 + * puts the "Per-Message Compressed" bit, RSV1, on the first + * fragment of a compressed data message), so its bytes are not + * text until the application has decoded them. + */ + h1p->websocket_text = (wsh->opcode == NXT_WEBSOCKET_OP_TEXT + && wsh->rsv1 == 0 + && wsh->rsv2 == 0 + && wsh->rsv3 == 0); + nxt_memzero(&h1p->websocket_utf8, sizeof(nxt_h1p_ws_utf8_t)); } h1p->websocket_cont_expected = !wsh->fin; @@ -410,13 +436,190 @@ nxt_h1p_conn_ws_keepalive_enable(nxt_task_t *task, nxt_h1proto_t *h1p) } +/* + * Returns the length of the UTF-8 sequence a lead byte announces, or 0 when + * the byte cannot begin one. C0 and C1 have only overlong two-byte + * encodings and F5..FF are beyond U+10FFFF, so neither can lead a sequence + * (RFC 3629 Section 3). Shortest form, surrogates, and the byte ranges that + * apply only after E0, ED, F0 and F4 are left to nxt_utf8_decode(). + */ +static size_t +nxt_h1p_ws_utf8_seqlen(u_char c) +{ + if (c >= 0xF0) { + return (c <= 0xF4) ? 4 : 0; + } + + if (c >= 0xE0) { + return 3; + } + + return (c >= 0xC2) ? 2 : 0; +} + + +/* + * Decides whether the "n" bytes at "seq", the whole or the beginning of one + * UTF-8 sequence, can be a valid character: NXT_OK when they already are, + * NXT_AGAIN when a continuation is still needed, NXT_ERROR when no byte + * following them can repair the sequence. + * + * A partial sequence is completed with the smallest continuation bytes a + * valid encoding could have and handed to nxt_utf8_decode(), so that this + * test cannot drift from the decoder that finally accepts the bytes: E0 must + * be followed by A0..BF, F0 by 90..BF and F4 by 80..8F, or the character is + * overlong or beyond U+10FFFF however the sequence ends. + */ +static nxt_int_t +nxt_h1p_ws_utf8_begin(const u_char *seq, size_t n) +{ + u_char buf[4]; + size_t len; + const u_char *p; + + len = nxt_h1p_ws_utf8_seqlen(seq[0]); + + if (nxt_slow_path(len == 0)) { + return NXT_ERROR; + } + + if (n < len) { + nxt_memcpy(buf, seq, n); + + if (n == 1) { + switch (seq[0]) { + case 0xE0: + buf[n++] = 0xA0; + break; + case 0xF0: + buf[n++] = 0x90; + break; + default: + buf[n++] = 0x80; + } + } + + while (n < 4) { + buf[n++] = 0x80; + } + + p = buf; + + if (nxt_utf8_decode(&p, buf + 4) == 0xFFFFFFFF) { + return NXT_ERROR; + } + + return NXT_AGAIN; + } + + p = seq; + + if (nxt_utf8_decode(&p, seq + n) == 0xFFFFFFFF) { + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Validates the payload of a data or close frame as UTF-8. "length" bytes + * are read from "start", the first payload byte in "b"; the rest of the + * frame, if it did not fit in one buffer, follows in the "next" chain. The + * payload is raw, so every byte is unmasked with mask[i % 4] as it is read, + * "i" being its offset from the first byte of the payload. "skip" bytes + * before "start" are stepped over: they are the close code, which is not + * text. + * + * A sequence the end of the frame cut short is kept in "state" and completed + * by the frame that continues the message. With "final" set the payload + * must end on a character boundary, as a message and a close reason do. + */ +static nxt_int_t +nxt_h1p_ws_utf8_validate(nxt_h1p_ws_utf8_t *state, nxt_buf_t *b, u_char *start, + size_t skip, size_t length, const u_char *mask, nxt_uint_t final) +{ + u_char c, seq[4]; + size_t i, n; + const u_char *p, *end; + nxt_buf_t *in; + + in = b; + p = start; + end = in->mem.free; + + for (i = 0; i < skip + length; /* void */) { + + while (nxt_slow_path(p >= end)) { + in = in->next; + + if (nxt_slow_path(in == NULL)) { + /* + * The frame is shorter than its header says, which cannot + * happen: a frame is buffered whole before it is handled + * here. Answered apart from invalid UTF-8 so that a reader + * fault is not reported as an encoding one. + */ + return NXT_DECLINED; + } + + p = in->mem.start; + end = in->mem.free; + } + + c = *p++ ^ mask[i % 4]; + i++; + + if (nxt_slow_path(i <= skip)) { + continue; + } + + if (nxt_fast_path(state->len == 0 && c < 0x80)) { + continue; + } + + nxt_memcpy(seq, state->tail, state->len); + seq[state->len] = c; + n = state->len + 1; + + switch (nxt_h1p_ws_utf8_begin(seq, n)) { + + case NXT_OK: + state->len = 0; + break; + + case NXT_AGAIN: + if (nxt_slow_path(n > sizeof(state->tail))) { + /* Cannot be taken: NXT_AGAIN means n < len <= 4. */ + return NXT_ERROR; + } + + nxt_memcpy(state->tail, seq, n); + state->len = n; + break; + + default: + return NXT_ERROR; + } + } + + if (nxt_slow_path(final && state->len != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + static void nxt_h1p_conn_ws_frame_process(nxt_task_t *task, nxt_conn_t *c, nxt_h1proto_t *h1p, nxt_websocket_header_t *wsh) { - size_t hsize; + size_t hsize, payload_len; uint8_t *p, *mask; uint16_t code; + nxt_int_t res; + nxt_h1p_ws_utf8_t utf8; nxt_http_request_t *r; r = h1p->request; @@ -444,9 +647,74 @@ nxt_h1p_conn_ws_frame_process(nxt_task_t *task, nxt_conn_t *c, code); return; } + + /* + * RFC 6455 Section 5.5.1: the reason is UTF-8-encoded. It does + * not continue the message in progress, so it is validated + * against its own state and must be complete. Unlike a data + * frame it is read with an RSV bit set: permessage-deflate and + * its relatives apply to data messages only (RFC 7692 + * Section 6), and the 2-byte code above is read the same way. + */ + if (wsh->payload_len > 2) { + payload_len = (size_t) nxt_websocket_frame_payload_len(wsh); + + nxt_memzero(&utf8, sizeof(nxt_h1p_ws_utf8_t)); + + res = nxt_h1p_ws_utf8_validate(&utf8, r->ws_frame, p, 2, + payload_len - 2, mask, 1); + + if (nxt_slow_path(res != NXT_OK)) { + hxt_h1p_send_ws_error(task, r, + (res == NXT_DECLINED) + ? &nxt_ws_err_truncated + : &nxt_ws_err_invalid_utf8); + return; + } + } } h1p->websocket_closed = 1; + nxt_memzero(&h1p->websocket_utf8, sizeof(nxt_h1p_ws_utf8_t)); + + } else if (h1p->websocket_text + && (wsh->opcode == NXT_WEBSOCKET_OP_TEXT + || wsh->opcode == NXT_WEBSOCKET_OP_CONT)) + { + if (nxt_slow_path(wsh->rsv1 != 0 || wsh->rsv2 != 0 + || wsh->rsv3 != 0)) + { + /* + * Some extension defines this frame (RFC 6455 Section 5.2), so + * neither it nor what follows in the message can be read as + * text here. + */ + h1p->websocket_text = 0; + + } else { + /* + * RFC 6455 Section 5.6: a text message is UTF-8 text however + * many frames it is split into, and Section 8.1 fails the + * connection on data it cannot interpret; Section 7.4.1 gives + * the close code for that, 1007. Binary messages are not text + * and are passed on unread. + */ + hsize = nxt_websocket_frame_header_size(wsh); + mask = nxt_pointer_to(wsh, hsize - 4); + payload_len = (size_t) nxt_websocket_frame_payload_len(wsh); + + res = nxt_h1p_ws_utf8_validate(&h1p->websocket_utf8, r->ws_frame, + nxt_pointer_to(wsh, hsize), 0, + payload_len, mask, wsh->fin); + + if (nxt_slow_path(res != NXT_OK)) { + hxt_h1p_send_ws_error(task, r, + (res == NXT_DECLINED) + ? &nxt_ws_err_truncated + : &nxt_ws_err_invalid_utf8); + return; + } + } } r->state->ready_handler(task, r, NULL); diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index 73102fa76..191bc9dfa 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -433,9 +433,8 @@ def test_asgi_websockets_protocol_absent(): # autobahn-testsuite # -# Some following tests fail because of Unit does not support UTF-8 -# validation for websocket frames. It should be implemented -# by application, if necessary. +# The router validates the payload of a text message and the reason of a +# close frame as UTF-8 and fails the connection with 1007 when it is not. def test_asgi_websockets_1_1_1__1_1_8(): @@ -1106,28 +1105,35 @@ def test_asgi_websockets_6_1_1__6_4_4(): close_connection(sock) + # 6_3_1 -# Unit does not support UTF-8 validation -# -# # 6_3_1 FAIL -# -# payload_1 = '\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' -# payload_2 = '\xed\xa0\x80' -# payload_3 = '\x65\x64\x69\x74\x65\x64' -# -# payload = payload_1 + payload_2 + payload_3 -# -# ws.message(sock, ws.OP_TEXT, payload) -# check_close(sock, 1007) -# -# # 6_3_2 FAIL -# -# _, sock, _ = ws.upgrade() -# -# ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) -# check_close(sock, 1007) -# -# # 6_4_1 ... 6_4_4 FAIL + _, sock, _ = ws.upgrade() + + # Raw bytes: written as str these would be encoded to valid UTF-8. + payload_1 = b'\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' + payload_2 = b'\xed\xa0\x80' + payload_3 = b'\x65\x64\x69\x74\x65\x64' + + payload = payload_1 + payload_2 + payload_3 + + ws.message(sock, ws.OP_TEXT, payload) + check_close(sock, 1007) + + # 6_3_2 + + _, sock, _ = ws.upgrade() + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) + check_close(sock, 1007) + + # A text message whose last frame ends inside a multi-byte sequence: the + # frame is complete, the message is not valid UTF-8. + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, b'\xce') + + check_close(sock, 1007) def test_asgi_websockets_7_1_1__7_5_1(): @@ -1255,16 +1261,26 @@ def test_asgi_websockets_7_1_1__7_5_1(): ws.frame_write(sock, ws.OP_CLOSE, payload) check_close(sock, 1002) + # 7_5_1 -# # 7_5_1 FAIL Unit does not support UTF-8 validation -# -# _, sock, _ = ws.upgrade() -# -# payload = ws.serialize_close(reason = '\xce\xba\xe1\xbd\xb9\xcf' \ -# '\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64') -# -# ws.frame_write(sock, ws.OP_CLOSE, payload) -# check_close(sock, 1007) + _, sock, _ = ws.upgrade() + + # A raw reason: 0xed 0xa0 0x80 is a surrogate, which has no valid UTF-8 + # encoding. + payload = ws.serialize_close() + b'\xce\xba\xe1\xbd\xb9\xcf' \ + b'\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64' + + ws.frame_write(sock, ws.OP_CLOSE, payload) + check_close(sock, 1007) + + # A close reason that is valid UTF-8 but not ASCII is accepted. + + _, sock, _ = ws.upgrade() + + payload = ws.serialize_close() + b'\xce\xba' + + ws.frame_write(sock, ws.OP_CLOSE, payload) + check_close(sock) def test_asgi_websockets_7_7_X__7_9_X(): diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 031d6924f..fe0bb1bad 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -349,9 +349,8 @@ def test_java_websockets_protocol_absent(): # autobahn-testsuite # -# Some following tests fail because of Unit does not support UTF-8 -# validation for websocket frames. It should be implemented -# by application, if necessary. +# The router validates the payload of a text message and the reason of a +# close frame as UTF-8 and fails the connection with 1007 when it is not. def test_java_websockets_1_1_1__1_1_8(): @@ -1022,28 +1021,26 @@ def test_java_websockets_6_1_1__6_4_4(): close_connection(sock) + # 6_3_1 -# Unit does not support UTF-8 validation -# -# # 6_3_1 FAIL -# -# payload_1 = '\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' -# payload_2 = '\xed\xa0\x80' -# payload_3 = '\x65\x64\x69\x74\x65\x64' -# -# payload = payload_1 + payload_2 + payload_3 -# -# ws.message(sock, ws.OP_TEXT, payload) -# check_close(sock, 1007) -# -# # 6_3_2 FAIL -# -# _, sock, _ = ws.upgrade() -# -# ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) -# check_close(sock, 1007) -# -# # 6_4_1 ... 6_4_4 FAIL + _, sock, _ = ws.upgrade() + + # Raw bytes: written as str these would be encoded to valid UTF-8. + payload_1 = b'\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' + payload_2 = b'\xed\xa0\x80' + payload_3 = b'\x65\x64\x69\x74\x65\x64' + + payload = payload_1 + payload_2 + payload_3 + + ws.message(sock, ws.OP_TEXT, payload) + check_close(sock, 1007) + + # 6_3_2 + + _, sock, _ = ws.upgrade() + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) + check_close(sock, 1007) def test_java_websockets_7_1_1__7_5_1(): @@ -1171,16 +1168,17 @@ def test_java_websockets_7_1_1__7_5_1(): ws.frame_write(sock, ws.OP_CLOSE, payload) check_close(sock, 1002) + # 7_5_1 -# # 7_5_1 FAIL Unit does not support UTF-8 validation -# -# _, sock, _ = ws.upgrade() -# -# payload = ws.serialize_close(reason = '\xce\xba\xe1\xbd\xb9\xcf' \ -# '\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64') -# -# ws.frame_write(sock, ws.OP_CLOSE, payload) -# check_close(sock, 1007) + _, sock, _ = ws.upgrade() + + # A raw reason: 0xed 0xa0 0x80 is a surrogate, which has no valid UTF-8 + # encoding. + payload = ws.serialize_close() + b'\xce\xba\xe1\xbd\xb9\xcf' \ + b'\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64' + + ws.frame_write(sock, ws.OP_CLOSE, payload) + check_close(sock, 1007) def test_java_websockets_7_7_X__7_9_X(): diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 26baca07a..44fd090f4 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -369,9 +369,8 @@ def test_node_websockets_protocol_absent(): # autobahn-testsuite # -# Some following tests fail because of Unit does not support UTF-8 -# validation for websocket frames. It should be implemented -# by application, if necessary. +# The router validates the payload of a text message and the reason of a +# close frame as UTF-8 and fails the connection with 1007 when it is not. def test_node_websockets_1_1_1__1_1_8(): @@ -1042,28 +1041,26 @@ def test_node_websockets_6_1_1__6_4_4(): close_connection(sock) + # 6_3_1 -# Unit does not support UTF-8 validation -# -# # 6_3_1 FAIL -# -# payload_1 = '\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' -# payload_2 = '\xed\xa0\x80' -# payload_3 = '\x65\x64\x69\x74\x65\x64' -# -# payload = payload_1 + payload_2 + payload_3 -# -# ws.message(sock, ws.OP_TEXT, payload) -# check_close(sock, 1007) -# -# # 6_3_2 FAIL -# -# _, sock, _ = ws.upgrade() -# -# ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) -# check_close(sock, 1007) -# -# # 6_4_1 ... 6_4_4 FAIL + _, sock, _ = ws.upgrade() + + # Raw bytes: written as str these would be encoded to valid UTF-8. + payload_1 = b'\xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xb5' + payload_2 = b'\xed\xa0\x80' + payload_3 = b'\x65\x64\x69\x74\x65\x64' + + payload = payload_1 + payload_2 + payload_3 + + ws.message(sock, ws.OP_TEXT, payload) + check_close(sock, 1007) + + # 6_3_2 + + _, sock, _ = ws.upgrade() + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=1) + check_close(sock, 1007) def test_node_websockets_7_1_1__7_5_1(): @@ -1191,16 +1188,17 @@ def test_node_websockets_7_1_1__7_5_1(): ws.frame_write(sock, ws.OP_CLOSE, payload) check_close(sock, 1002) + # 7_5_1 -# # 7_5_1 FAIL Unit does not support UTF-8 validation -# -# _, sock, _ = ws.upgrade() -# -# payload = ws.serialize_close(reason = '\xce\xba\xe1\xbd\xb9\xcf' \ -# '\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64') -# -# ws.frame_write(sock, ws.OP_CLOSE, payload) -# check_close(sock, 1007) + _, sock, _ = ws.upgrade() + + # A raw reason: 0xed 0xa0 0x80 is a surrogate, which has no valid UTF-8 + # encoding. + payload = ws.serialize_close() + b'\xce\xba\xe1\xbd\xb9\xcf' \ + b'\x83\xce\xbc\xce\xb5\xed\xa0\x80\x65\x64\x69\x74\x65\x64' + + ws.frame_write(sock, ws.OP_CLOSE, payload) + check_close(sock, 1007) def test_node_websockets_7_7_X__7_9_X(): From 4451fef4cf7e75ef14431a1359e67d0a2f8eb3e1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 20:45:08 +0200 Subject: [PATCH 211/391] port: keep the queue wake-up when the kernel could not send it A payload that fits the shared ring goes into the ring, and the socket write that follows is only the READ_QUEUE marker that tells the peer to look. nxt_port_queue_send() raises that marker on the 0-to-1 transition alone, so if it is lost nothing raises another: the peer stays unaware of that message and of everything queued on the port after it. nxt_socketpair_send() mapped EAGAIN and ENOBUFS to NXT_AGAIN and folded every other errno into NXT_ERROR. A kernel ENOMEM -- the same transient condition as ENOBUFS, and what Linux answers when the SCM_RIGHTS copy of a descriptor cannot be allocated -- therefore took the dead-peer path in nxt_port_write_msgs(): the marker was dropped, the error handler was raised against a live peer, and the write answered NXT_OK with nothing owed. ENOMEM now joins ENOBUFS as NXT_AGAIN. The two callers, both in src/nxt_port_socket.c, already handle it: nxt_port_write_msgs() queues the marker for a later attempt or reports it owed, and nxt_port_announce() leaves the count for the next pass. sendmsg() returned -1 either way, so no bytes and no descriptor went out on either answer; the "nothing consumed" contract of the NXT_ERROR exit is unchanged, and that exit now covers only errors the socket will not recover from. Keeping the marker is half of it. NXT_AGAIN clears write_ready, and the retry then waits for the write event. nxt_port_fd_block_write() left that event blocked after a drain, and enabling a blocked event touches no epoll registration; edge-triggered, a socket that never filled up raises no edge, so the queued marker waited on an event that would not fire. The drain now disables the event instead, and the re-arm registers it again, which has the poller re-check readiness on the spot. Measured on a real epoll engine before the change: one poll after the re-arm delivered nothing. Only the event-loop pass of nxt_port_write_msgs() asks for the block, and it runs on port->engine, so nxt_port_post() calls the handler directly there; the allocating post is not reached from that site. That re-arm covers the first retry only. It works because the drained pass left the event disabled and the inline pass enables a disabled event. The retry itself runs on the event-loop pass, and when it fails with ENOMEM again the event is already active: nothing enables an active event, the socket never filled so no edge is coming, and the message stays queued for ever with every later message on the port behind it -- since #389, dup'd descriptors among them. nxt_socketpair_send() now records the errno in the port socket's unused ->error before answering NXT_AGAIN, and the event-loop pass forces the readiness re-check the drained pass does -- disable, then enable -- whenever the cause was memory rather than a full socket. The cost is one epoll_ctl() pair per failed retry; under sustained ENOMEM that is one retry per poll, which is what the owed-marker path already does. Found in review with a two-shot version of the injection hook; the hook now takes a count. The test runs on that engine: ENOMEM injected on the wake-up, with the write event unregistered and then after a drain has taken it down; ENOMEM on the wake-up and again on the retry, where only the re-check brings the marker out; ENOMEM with no memory to queue the marker; and EPIPE, where nothing may be owed. The injection hook in nxt_socketpair_send() compiles only with NXT_TESTS. Closes #392. Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Fable 5.1 --- src/nxt_port_socket.c | 74 ++++- src/nxt_socket.h | 4 + src/nxt_socketpair.c | 74 ++++- src/test/nxt_port_fail_test.c | 493 ++++++++++++++++++++++++++++++++++ 4 files changed, 632 insertions(+), 13 deletions(-) diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 0c99a9453..081f15bd7 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -385,19 +385,20 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, * drains the ring before it reads the socket, so the next read pass * takes this item too. * - * ENOBUFS promises nothing: nxt_socketpair_send() maps it to - * NXT_AGAIN as well (src/nxt_socketpair.c), and it means the system - * ran out of buffers, not that the peer owes this port a read. + * ENOBUFS and ENOMEM promise nothing: nxt_socketpair_send() maps + * both to NXT_AGAIN as well (src/nxt_socketpair.c), and they mean + * the system could not allocate for the call, not that the peer + * owes this port a read. * * So record that a marker is owed and re-arm. nxt_port_announce() * sends it from the stack with no allocation, either here on this * port's own engine or from the item nxt_port_rearm() posts. * * Unless the send failed outright. Then it did not hit a transient - * condition: nxt_socketpair_send() answers NXT_AGAIN for EAGAIN and - * ENOBUFS and retries EINTR, so everything that reaches here is an - * error that will not get better -- a dead peer routinely, and - * EMSGSIZE, EBADF or a kernel ENOMEM otherwise. Every one of them + * condition: nxt_socketpair_send() answers NXT_AGAIN for EAGAIN, + * ENOBUFS and ENOMEM and retries EINTR, so everything that reaches + * here is an error that will not get better -- a dead peer + * routinely, and EMSGSIZE or EBADF otherwise. Every one of them * ends this port: the marker can never be delivered, and re-arming * the write event only brings this function back to fail the same * send, leaving another queued error handler each pass while the @@ -599,10 +600,43 @@ nxt_port_msg_dup_fds(nxt_port_send_msg_t *msg) } +/* + * Disable, not block. Blocking leaves the descriptor registered and only + * has the poller ignore what it reports; in edge-triggered mode that report + * is then lost, and the enable that later re-arms a blocked event touches + * the registration not at all. That is fine after EAGAIN, where the + * socket is full and the peer's next read raises a fresh edge. After a + * send that failed for want of kernel memory it is not: the socket stayed + * writable, no edge is coming, and the message queued for a later attempt + * would wait on an event that never fires. Disabling drops the + * registration, so the re-arm adds it back and the poller re-checks + * readiness on the spot (#392). + * + * The cost is one epoll_ctl() here and one on the next EAGAIN, both off + * the inline path. Only the event-loop pass of nxt_port_write_msgs() -- + * data == NULL, the queue drained -- asks for this, and that pass runs on + * port->engine, where nxt_port_post() calls the handler directly: the + * post that allocates, and whose failure would skip this, is not reached + * from here. + * + * Both tests below are repeated from the caller, for the same reason + * nxt_port_announce() repeats its own: the descriptor, because a port can + * be closed while this item is in flight and nxt_port_write_close() leaves + * socket.fd at the number it had, so the disable would go out on a closed + * or already reused descriptor (see nxt_port_rearm_now()); the event state, + * because nxt_fd_event_disable_write() carries no guard of its own, unlike + * the nxt_fd_event_block_write() this replaced, and an EV_DISABLE on a + * kqueue knote that was never added is ENOENT. + */ + static void -nxt_port_fd_block_write(nxt_task_t *task, nxt_port_t *port, void *data) +nxt_port_fd_disable_write(nxt_task_t *task, nxt_port_t *port, void *data) { - nxt_fd_event_block_write(task->thread->engine, &port->socket); + if (port->pair[1] == -1 || !nxt_fd_event_is_active(port->socket.write)) { + return; + } + + nxt_fd_event_disable_write(task->thread->engine, &port->socket); } @@ -976,6 +1010,26 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, goto fail; } + /* + * NXT_AGAIN. After EAGAIN the socket is full, and the peer's + * next read raises the edge that brings this handler back. + * After ENOBUFS or ENOMEM it is not full: the kernel could not + * allocate for the call, and no edge is coming. The inline + * pass re-arms a disabled event, which is how the first retry + * runs at all; but this pass finds the event active, and an + * active edge-triggered event is never re-added, so a retry + * that runs out of memory again would leave the message queued + * for ever, and every later message on this port behind it. + * Force the readiness re-check the drained pass does -- disable, + * then enable -- so the poller reports the socket writable on + * the spot. One epoll_ctl() pair per failed retry. + */ + + if (data == NULL && port->socket.error != NXT_EAGAIN) { + block_write = 1; + enable_write = 1; + } + if (msg->link.next == NULL) { qmsg = nxt_port_msg_insert_tail(port, msg); if (nxt_slow_path(qmsg == NULL)) { @@ -1040,7 +1094,7 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, cleanup: if (block_write && nxt_fd_event_is_active(port->socket.write)) { - nxt_port_post(task, port, nxt_port_fd_block_write, NULL); + nxt_port_post(task, port, nxt_port_fd_disable_write, NULL); } /* diff --git a/src/nxt_socket.h b/src/nxt_socket.h index 69b09039f..6467e31a1 100644 --- a/src/nxt_socket.h +++ b/src/nxt_socket.h @@ -117,6 +117,10 @@ NXT_EXPORT ssize_t nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, NXT_EXPORT ssize_t nxt_socketpair_recv(nxt_fd_event_t *ev, nxt_iobuf_t *iob, nxt_uint_t niob, void *oob); +#if (NXT_TESTS) +NXT_EXPORT void nxt_socketpair_test_send_fail(nxt_err_t err, nxt_uint_t n); +#endif + #define nxt_socket_nonblocking(task, fd) \ nxt_fd_nonblocking(task, fd) diff --git a/src/nxt_socketpair.c b/src/nxt_socketpair.c index 45274b788..94ccf9984 100644 --- a/src/nxt_socketpair.c +++ b/src/nxt_socketpair.c @@ -82,6 +82,29 @@ nxt_socketpair_close(nxt_task_t *task, nxt_socket_t *pair) } +#if (NXT_TESTS) +static nxt_err_t nxt_socketpair_test_send_err; +static nxt_uint_t nxt_socketpair_test_send_err_left; + + +/* + * Make the next "n" calls of sendmsg() in nxt_socketpair_send() fail with + * "err" without running it. The call after that runs the real sendmsg(). + * Two is what expresses "the retry fails too", and zero injects nothing. + * src/test/nxt_port_fail_test.c uses it for the errors a socketpair between + * two live processes will not produce on demand. + */ + +void +nxt_socketpair_test_send_fail(nxt_err_t err, nxt_uint_t n) +{ + nxt_socketpair_test_send_err = err; + nxt_socketpair_test_send_err_left = (err != 0) ? n : 0; +} + +#endif + + ssize_t nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) @@ -93,14 +116,41 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_socket_msg_oob_init(&oob, fd); for ( ;; ) { - n = nxt_sendmsg(ev->fd, iob, niob, &oob); + n = -1; + err = 0; - err = (n == -1) ? nxt_socket_errno : 0; +#if (NXT_TESTS) + if (nxt_socketpair_test_send_err_left > 0) { + err = nxt_socketpair_test_send_err; + + nxt_socketpair_test_send_err_left--; + + if (nxt_socketpair_test_send_err_left == 0) { + nxt_socketpair_test_send_err = 0; + } + } +#endif + + if (nxt_fast_path(err == 0)) { + n = nxt_sendmsg(ev->fd, iob, niob, &oob); + + err = (n == -1) ? nxt_socket_errno : 0; + } nxt_debug(ev->task, "sendmsg(%d, %FD, %FD, %ui): %z", ev->fd, fd[0], fd[1], niob, n); if (n > 0) { + /* + * Everywhere else in the tree a non-zero socket.error means the + * socket is broken (nxt_conn_write.c, nxt_conn_read.c, + * nxt_conn_close.c), and nxt_conn_io_write() stores 0 on a + * successful write for that reason. Keep the field meaning the + * same thing here: the errno recorded below is only for the + * NXT_AGAIN that follows it. + */ + ev->error = 0; + return n; } @@ -114,9 +164,19 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, /* * Returned (at least on OSX) when trying to send many small messages. + * + * ENOMEM is the same condition by another name: the kernel could not + * allocate for this call. Linux answers it when the SCM_RIGHTS + * copy of a descriptor-carrying message cannot be allocated, and + * nothing of the message has gone out. The socket and the peer are + * fine, so the caller must keep the message and try again, as it + * does after ENOBUFS. The "default" arm below is for errors the + * socket will not recover from; answering NXT_ERROR here made a + * caller drop a message a live peer was still owed (#392). */ case NXT_ENOBUFS: - nxt_debug(ev->task, "sendmsg(%d) no buffers", ev->fd); + case NXT_ENOMEM: + nxt_debug(ev->task, "sendmsg(%d) cannot allocate %E", ev->fd, err); break; case NXT_EINTR: @@ -127,9 +187,17 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_alert(ev->task, "sendmsg(%d, %FD, %FD, %ui) failed %E", ev->fd, fd[0], fd[1], niob, err); + /* + * The one exit where the socket really is broken, so it is the + * one that must leave the errno behind: see the note above on + * what a non-zero socket.error means. + */ + ev->error = err; + return NXT_ERROR; } + ev->error = err; ev->write_ready = 0; return NXT_AGAIN; diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index f416f3ab7..b1212f018 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -19,6 +19,12 @@ static nxt_int_t nxt_port_fail_test_socket_write(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_inline_drop(nxt_thread_t *thr); static void nxt_port_fail_test_enable_write(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static nxt_int_t nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr); +#if (NXT_HAVE_EPOLL_EDGE) +static nxt_int_t nxt_port_fail_test_enqueue(nxt_task_t *task, + nxt_port_t *port, nxt_mp_t *mp, nxt_err_t err, nxt_uint_t fails, + nxt_bool_t no_memory, nxt_bool_t *queued); +#endif static nxt_int_t nxt_port_fail_test_dead_peer(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_error_handler(nxt_thread_t *thr); @@ -70,6 +76,10 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_wakeup_errno(thr) != NXT_OK) { + return NXT_ERROR; + } + if (nxt_port_fail_test_rpc_register(thr) != NXT_OK) { return NXT_ERROR; } @@ -2275,3 +2285,486 @@ nxt_port_fail_test_fd_count(void) return -1; #endif } + + +/* + * One payload into the shared ring, with the wake-up that follows failing + * with "err" -- and with no memory to queue the marker when "no_memory" is + * set. The answer must be NXT_OK and the payload completed exactly once + * whatever became of the wake-up: it is in the ring, and the caller must + * not be told to reclaim it. The ring is drained first so that this + * enqueue is the 0-to-1 transition that raises the wake-up at all. + */ + +#if (NXT_HAVE_EPOLL_EDGE) + +static nxt_int_t +nxt_port_fail_test_enqueue(nxt_task_t *task, nxt_port_t *port, nxt_mp_t *mp, + nxt_err_t err, nxt_uint_t fails, nxt_bool_t no_memory, nxt_bool_t *queued) +{ + nxt_buf_t *buf; + nxt_int_t ret; + u_char block[NXT_PORT_QUEUE_MSG_SIZE]; + + while (nxt_port_queue_recv(port->queue, block) > 0) { + /* void */ + } + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + return NXT_ERROR; + } + + buf->completion_handler = nxt_port_fail_test_completion; + buf->mem.free++; + + port->socket.write_ready = 1; + + nxt_socketpair_test_send_fail(err, fails); + + if (no_memory) { + nxt_port_test_msg_alloc_failures(1); + } + + ret = nxt_port_socket_write(task, port, NXT_PORT_MSG_DATA, -1, 0, 0, buf); + + nxt_port_test_msg_alloc_failures(0); + + /* + * The send hook clears itself once its count is spent, and the inline + * write above spends one. A leg that arms more than one clears what + * is left when it is done, so that a retry can be made to fail too. + */ + + if (ret != NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: a payload that reached the shared " + "queue answered %d after sendmsg() error %d, expected " + "NXT_OK", (int) ret, (int) err); + return NXT_ERROR; + } + + /* + * Read before the drain, because the drain destroys the answer: an + * error raised by the write queues nxt_port_error_handler(), which + * empties port->messages whatever is in it. A caller that tested the + * queue after this returned would find it empty either way. + */ + + if (queued != NULL) { + *queued = !nxt_queue_is_empty(&port->messages); + } + + nxt_port_fail_test_drain_wq(&task->thread->engine->fast_work_queue); + + if (nxt_port_fail_test_completions != 1) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: the enqueued payload was completed " + "%d times after sendmsg() error %d, expected once", + (int) nxt_port_fail_test_completions, (int) err); + return NXT_ERROR; + } + + return NXT_OK; +} + +#endif + + +/* + * The wake-up that fails with an errno the socket recovers from (#392). + * + * A payload that fits the shared ring goes in there, and the inline write + * is only the READ_QUEUE marker that tells the peer to look. + * nxt_port_queue_send() raises "notify" on the 0-to-1 transition alone, so + * if that one marker is lost, no later enqueue on the port raises another: + * the peer stays unaware of this message and of everything after it. + * + * nxt_socketpair_send() answers NXT_AGAIN for EAGAIN and ENOBUFS, and + * nxt_port_write_msgs() then keeps the marker -- queued on the port, or + * owed in port->announce when even that allocation fails. A kernel ENOMEM + * is the same transient condition, and used to fall into the arm for + * errors the socket does not recover from: the marker was dropped, the + * error handler was raised against a live peer, and the write answered + * NXT_OK with nothing owed. + * + * NXT_AGAIN also clears write_ready, so keeping the marker is only half of + * it: the re-arm has to bring the write handler back on its own, with no + * later write from a caller to prompt it. So this runs on a real epoll + * engine, edge-triggered as the product's is, and the marker is read off + * the peer's end after one poll. Twice: once with the write event never + * armed, and once after the drain that sent the first marker has taken + * the event down again -- which used to leave it blocked rather than + * disabled, so the second re-arm touched no registration and the marker + * waited on an edge that never came. + * + * Then the same ENOMEM with no memory to queue the marker, which must be + * owed and announced from the stack; and EPIPE, where the peer is gone and + * nothing may be owed or armed. The errno is injected through + * nxt_socketpair_test_send_fail(), since a live pair does not produce + * either on demand. + */ + +static nxt_int_t +nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) +{ +#if (NXT_HAVE_EPOLL_EDGE) + ssize_t n; + nxt_mp_t *mp; + nxt_fd_t pair[2]; + nxt_int_t ret; + nxt_uint_t i; + nxt_bool_t queued; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t *engine, *saved_engine; + nxt_port_queue_t *queue; + nxt_port_send_msg_t *msg; + u_char block[NXT_PORT_QUEUE_MSG_SIZE]; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + pair[0] = -1; + pair[1] = -1; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + saved_engine = thr->engine; + + engine = nxt_event_engine_create(task, &nxt_epoll_edge_engine, NULL, 0, 0); + if (nxt_slow_path(engine == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + thr->engine = engine; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + goto fail; + } + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: socketpair failed"); + goto done; + } + + if (nxt_slow_path(fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: O_NONBLOCK failed"); + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.task = task; + port->max_size = 1024; + port->max_share = 1024; + + /* The real thing: handlers, engine, write_ready, and no event armed. */ + + nxt_port_write_enable(task, port); + + port->socket.log = thr->log; + + queue = nxt_mp_zalloc(mp, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue == NULL)) { + goto done; + } + + nxt_port_queue_init(queue); + port->queue = queue; + + /* + * Legs 1 and 2: ENOMEM on the wake-up, with memory to queue the marker. + * The first arms a write event that was never registered; the second + * finds it blocked by the drain that sent the first marker, and the + * enable then touches no epoll registration. Edge-triggered, the + * second is the one that would stall if the re-arm did not re-check. + */ + + for (i = 1; i <= 2; i++) { + + /* + * Quiescent first. The peer's read of the previous marker wakes + * this end's writers, and epoll keeps that as a ready entry for a + * registered descriptor; a poll here consumes it, so the leg + * measures the re-arm and not a wake-up left over from the last + * leg. + */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 1, 0, NULL) + != NXT_OK) + { + goto done; + } + + if (nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: ENOMEM on the wake-up " + "dropped the queue marker (announce %d); the peer " + "is never told to read the ring", i, + (int) port->announce); + goto done; + } + + msg = nxt_queue_link_data(nxt_queue_first(&port->messages), + nxt_port_send_msg_t, link); + + if (msg->port_msg.type != _NXT_PORT_MSG_READ_QUEUE) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: the message kept " + "after ENOMEM is of type %d, expected READ_QUEUE", + i, (int) msg->port_msg.type); + goto done; + } + + if (port->socket.write_ready != 0 + || !nxt_fd_event_is_active(port->socket.write)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: after ENOMEM " + "write_ready is %d and the write event state is " + "%d; expected 0 and an active event", i, + (int) port->socket.write_ready, + (int) port->socket.write); + goto done; + } + + /* One poll, nothing else: the re-arm alone must bring it back. */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != (ssize_t) sizeof(nxt_port_msg_t) + || ((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: the peer read %d bytes " + "after the re-arm and one poll, expected a %d byte " + "READ_QUEUE header (write event state %d)", i, + (int) n, (int) sizeof(nxt_port_msg_t), + (int) port->socket.write); + goto done; + } + + if (!nxt_queue_is_empty(&port->messages) || port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: the marker went out " + "but is still held (announce %d)", i, + (int) port->announce); + goto done; + } + + if (nxt_fd_event_is_active(port->socket.write)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg %ui: the drain left the " + "write event in state %d, expected it disabled", i, + (int) port->socket.write); + goto done; + } + } + + /* + * Leg 3: ENOMEM on the wake-up, and ENOMEM again when the write event + * retries the queued marker. The first failure is what legs 1 and 2 + * cover: the inline pass re-arms a disabled event and the retry runs + * on the next poll. The second finds the event already active, so + * nothing re-arms it, and the socket never filled, so no edge is + * coming either. The retry pass has to force the readiness re-check + * itself; without it the marker, and every message queued behind it, + * waits for ever. + */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 2, 0, NULL) + != NXT_OK) + { + goto done; + } + + if (nxt_queue_is_empty(&port->messages) || port->socket.write_ready != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: after the first ENOMEM the " + "marker is not queued, or write_ready is %d", + (int) port->socket.write_ready); + goto done; + } + + /* The first poll runs the retry, which is made to fail again. */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + if (nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: the retry did not fail; " + "the marker left the queue after one poll"); + goto done; + } + + /* The second poll has only the re-check to bring the retry back. */ + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + nxt_socketpair_test_send_fail(0, 0); + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != (ssize_t) sizeof(nxt_port_msg_t) + || ((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: the peer read %d bytes after " + "a retry that ran out of memory and two polls, expected " + "a %d byte READ_QUEUE header; the queued marker is " + "stranded (write event state %d, write_ready %d)", + (int) n, (int) sizeof(nxt_port_msg_t), + (int) port->socket.write, + (int) port->socket.write_ready); + goto done; + } + + if (!nxt_queue_is_empty(&port->messages) || port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: the marker went out but is " + "still held (announce %d)", (int) port->announce); + goto done; + } + + if (nxt_fd_event_is_active(port->socket.write)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 3: the drain left the write " + "event in state %d, expected it disabled", + (int) port->socket.write); + goto done; + } + + /* Leg 4: ENOMEM on the wake-up, and no memory to queue the marker. */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 1, 1, NULL) + != NXT_OK) + { + goto done; + } + + /* + * Owed, and sent from the stack by the re-arm inside the same write, + * before any poll: the socket is writable, so nxt_port_announce() runs + * on the spot and the count is back to 0 by the time the write answers. + */ + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: ENOMEM with no memory to queue the " + "marker queued one anyway"); + goto done; + } + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (port->announce != 0 || n != (ssize_t) sizeof(nxt_port_msg_t) + || ((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: the owed marker was not announced " + "(announce %d, %d bytes read)", (int) port->announce, + (int) n); + goto done; + } + + /* Leg 5: EPIPE. The peer is gone; nothing is owed and nothing sent. */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_EPIPE, 1, 0, &queued) + != NXT_OK) + { + goto done; + } + + if (queued || port->announce != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: EPIPE on the wake-up kept a marker " + "for a peer that is gone (queued %d, announce %d)", + (int) queued, (int) port->announce); + goto done; + } + + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: %d bytes reached the peer after " + "EPIPE", (int) n); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_socketpair_test_send_fail(0, 0); + nxt_port_test_msg_alloc_failures(0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + + /* See nxt_port_fail_test_inline_drop() for both of these. */ + + port->queue = NULL; + + nxt_port_close(task, port); + + if (pair[0] != -1 && nxt_test_fd_is_open(pair[0])) { + nxt_fd_close(pair[0]); + } + + nxt_port_use(task, port, -1); + +fail: + + thr->engine = saved_engine; + + nxt_event_engine_free(engine); + nxt_mp_destroy(mp); + + return ret; + +#else + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: wake-up errno leg needs epoll, skipped"); + + return NXT_OK; + +#endif +} From c8a07534b1a3153becae3b09165b3d6957016dc5 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 20:45:08 +0200 Subject: [PATCH 212/391] changelog: the queue wake-up is kept when the kernel could not send it Co-Authored-By: Claude Opus 5 --- CHANGES | 8 ++++++++ docs/changes.xml | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/CHANGES b/CHANGES index fa5637d27..9b1d89e8a 100644 --- a/CHANGES +++ b/CHANGES @@ -23,6 +23,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 is not valid UTF-8 with U+FFFD before it serializes. *) Change: upgrade contrib njs to 1.0.1. + *) Bugfix: a process could stop hearing from a peer for good. When the + wake-up that tells a peer to read the shared queue failed because + the kernel could not allocate for it, the port treated the peer as + dead: the wake-up was dropped and, since a later message raises one + only when the queue was empty, no later message on that port raised + one either. The wake-up is now kept and retried, and the retry no + longer waits for a socket event that a socket that never filled up + will not deliver (#392). *) Security: a port message that was queued because the peer's socket was not writable borrowed the descriptors it carried -- for NEW_PORT, diff --git a/docs/changes.xml b/docs/changes.xml index 351cc21ac..bc2091025 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -42,6 +42,18 @@ send (#388). + + +a process could stop hearing from a peer for good. When the wake-up that +tells a peer to read the shared queue failed because the kernel could not +allocate for it, the port treated the peer as dead: the wake-up was dropped +and, since a later message raises one only when the queue was empty, no +later message on that port raised one either. The wake-up is now kept and +retried, and the retry no longer waits for a socket event that a socket that +never filled up will not deliver (#392). + + + upgrade contrib njs to 1.0.1. From b20aeb78bbb1243b0580f7ffcd106a2847e6a892 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 17 Sep 2026 20:17:41 +0200 Subject: [PATCH 213/391] php: remove the TrueAsync code path The module carried a TrueAsync prototype cherry-picked in April 2026 (b78e5a48, from EdmondDantes/052afd8), written against a zend_async_event_t API that was expected in PHP 8.5 core and did not ship. Every #if NXT_PHP_TRUEASYNC block was compiled out. The configure probe in auto/modules/php looks for zend_async_event_t in the PHP headers, which no released PHP has -- including the 8.5.4 headers on this box -- so NXT_PHP_TRUEASYNC was always 0 and a stock php.unit.so carries no *true_async* symbol. The code could not have compiled had the probe succeeded: it reads nxt_php_app_conf_t.async and .entrypoint, which that struct does not have, and calls nxt_php_extension_init() from a nxt_php_extension.c that does not exist anywhere in the tree. The unguarded entrypoint branch in nxt_php_set_target() was unreachable for the same reason: "entrypoint" and "async" are not php application members, and the validator answers "Unknown parameter" for both. Removed with it: the probe and its -D in auto/modules/php, the test suite test/test_php_trueasync.py with its three fixtures, the TrueAsync TODO section, and the mode split in nxt_php_start() with its "PHP standard mode" debug line. Every NULL-context guard that same commit added to the SAPI callbacks stays. The code in them is unchanged; only the comments move off the entrypoint and onto the states that still exist: * nxt_php_unbuffered_write(), nxt_php_send_headers(), nxt_php_register_variables() and nxt_php_read_cookies(): module startup, opcache.preload included, runs before the first request with SG(server_context) still NULL, and $_SERVER is created from script content rather than from request state. * nxt_php_read_post() and nxt_php_log_message(): fastcgi_finish_request() clears the request while PHP keeps running, and request_parse_body() (PHP 8.4+) reaches the SAPI again afterwards. Without those checks nxt_unit_request_read() is handed a NULL request -- a probe shows PHP reading the content type out of the already freed request -- and php messages logged after fastcgi_finish_request() would go to the worker's stderr instead of the unit log. With the guards intact the six callbacks are code-identical to the base commit, so a php application behaves exactly as before. Kept as well: the released 1.35.2/1.35.3 changelog history, NXT_PHP_PRE_REQUEST_INIT (a PHP 8.5 SAPI field guard, unrelated to TrueAsync), and nxt_php_cleanup_targets() / nxt_php_targets_count. Measured on PHP 8.5.4 with preload scripts that write a marker, echo, call header() and use $_SERVER: identical behaviour before and after this change in every case. Co-Authored-By: DeepSeek V4.1 Flash Harness: DeepSeek Harness --- TODO.md | 125 ---- auto/modules/php | 26 - src/nxt_php_sapi.c | 960 +------------------------ test/php/async_mirror/entrypoint.php | 20 - test/php/async_shutdown/entrypoint.php | 22 - test/php/async_slow/entrypoint.php | 41 -- test/test_php_trueasync.py | 564 --------------- 7 files changed, 15 insertions(+), 1743 deletions(-) delete mode 100644 test/php/async_mirror/entrypoint.php delete mode 100644 test/php/async_shutdown/entrypoint.php delete mode 100644 test/php/async_slow/entrypoint.php delete mode 100644 test/test_php_trueasync.py diff --git a/TODO.md b/TODO.md index aa37926a9..4efc41686 100644 --- a/TODO.md +++ b/TODO.md @@ -47,7 +47,6 @@ **Open questions (decide before August):** -- [ ] PHP TrueAsync — determine source of `nxt_php_extension.c` (fork EdmondDantes or write from scratch) - [ ] OpenSSL 3.6 migration — verify clang-ast compatibility - [ ] Proxy request buffering (#58) — define scope (per-action vs global) @@ -318,130 +317,6 @@ In-repo audit (2026-05-31): --- -## PHP TrueAsync mode (branch: php-graceful-shutdown) - -Items below must be resolved before the branch can be merged and before -`test/test_php_trueasync.py` can run in CI. - -### 1. `nxt_php_app_conf_t` is missing `async` and `entrypoint` fields - -**Problem.** `src/nxt_php_sapi.c` (commit 58ffa236) references `c->async` -and `c->entrypoint` where `c` is a pointer to `nxt_php_app_conf_t` -(defined in `src/nxt_application.h`). The struct currently has only -`targets` and `options`, so the code **will not compile** as-is. - -The original upstream commit (EdmondDantes/052afd8) likely also modified -`nxt_application.h` and the JSON config validator; those changes were not -included when the commit was cherry-picked into this branch. - -**Risk.** Build failure on the first C compile step; CI is entirely broken. - -**Fix.** -- Add the missing fields to `nxt_php_app_conf_t` in `src/nxt_application.h`: - ```c - typedef struct { - nxt_conf_value_t *targets; - nxt_conf_value_t *options; - nxt_bool_t async; /* true when TrueAsync mode requested */ - nxt_str_t entrypoint; /* path to the PHP entrypoint script */ - } nxt_php_app_conf_t; - ``` -- Wire them up in the config parser (find where other PHP config fields are - read from JSON — probably `nxt_router.c` or a PHP-specific conf handler). -- Add validation rules in `nxt_conf_validation.c` so the REST API rejects - `entrypoint` without `async: true` and vice-versa. - ---- - -### 2. `nxt_php_extension.c` / `nxt_php_extension_init()` are missing - -**Problem.** `PHP_MINIT_FUNCTION(nxt_php_ext)` calls -`nxt_php_extension_init()` (line 221 of `nxt_php_sapi.c`) and the async -request handler references `nxt_php_request_callback` (declared `extern` -at line 2187). Both symbols are expected to come from `nxt_php_extension.c`, -which does not exist in the repository. - -Without it the linker will fail and there is no PHP-land API for user -scripts to register a request handler. The `\Unit\Server::setHandler()` -calls in `test/php/async_*/entrypoint.php` are placeholders until the -real extension is in place. - -**Risk.** Linker failure; all TrueAsync tests will skip until resolved. - -**Fix.** -- Locate the file in the EdmondDantes fork or write it from scratch. - Minimum required: - - `zval *nxt_php_request_callback` global (zeroed at startup). - - `nxt_php_extension_init()` — registers `\Unit\Server` with a static - `setHandler(callable $cb)` that stores `$cb` into the global above. - - `\Unit\Request` class with methods `body()`, `query()`, `headers()`, - `respond(int $status, array $headers, string $body)`. -- Add the new `.c` file to the PHP module build rules in `auto/modules/php`. -- Document the final PHP API surface so entrypoint authors have a stable - contract. - ---- - -### 3. No timeout / force-kill fallback in `nxt_php_quit_handler` - -**Problem.** `nxt_php_quit_handler()` calls `ZEND_ASYNC_SHUTDOWN()` and -returns immediately. If user coroutines are stuck in a tight loop or -blocking I/O that the TrueAsync scheduler cannot interrupt, the worker -hangs forever. - -**Risk.** Silent hang in production; container runtime must SIGKILL instead. - -**Fix.** -- After `ZEND_ASYNC_SHUTDOWN()`, arm a timer for a configurable grace - period (default 30 s). On expiry call `exit(1)`. -- Expose as a PHP application config option: - ```json - { "type": "php", "async": true, "entrypoint": "server.php", - "shutdown_timeout": 30 } - ``` -- Add a test that verifies the force-kill fires when a coroutine refuses - to stop. - ---- - -### 4. Behavior of in-flight requests on shutdown is undefined - -**Problem.** It is not documented whether `ZEND_ASYNC_SHUTDOWN()` lets -active request coroutines run to completion or cancels them immediately. - -`test_php_trueasync_inflight_request_completes` assumes completion -semantics. If cancellation is the designed behavior, the test assertion -must be changed to `pytest.xfail`. - -**Action.** -- Read the TrueAsync scheduler shutdown semantics. -- Document the behavior. Update the test accordingly. - ---- - -### 5. Pending writes in `drain_queue` are silently abandoned on shutdown - -**Problem.** If `ZEND_ASYNC_SHUTDOWN()` fires while `drain_queue` is -non-empty, buffered response bytes are never sent and the client receives -a truncated body. - -**Fix.** Drain the queue synchronously in `nxt_php_quit_handler()` before -calling `ZEND_ASYNC_SHUTDOWN()`. - ---- - -### 6. Test fixtures use a placeholder PHP API - -The entrypoint scripts in `test/php/async_*/entrypoint.php` use -`\Unit\Server::setHandler()` and `\Unit\Request` — these are provisional -names. Once item 2 is resolved: -- Update all three fixture files to match the real class/method names. -- Replace the `_check_trueasync_available()` runtime probe in - `test_php_trueasync.py` with a proper `prerequisites` feature flag - (requires updating `unit/check/discover_available.py`). - ---- - ## OpenSSL 3.6 — test openssl-3.x branch Before the OpenSSL 3.6 migration can be considered fully validated: diff --git a/auto/modules/php b/auto/modules/php index d37300651..c4094561a 100644 --- a/auto/modules/php +++ b/auto/modules/php @@ -211,31 +211,6 @@ else fi -nxt_feature="PHP TrueAsync API (zend_async_event_t)" -nxt_feature_name="" -nxt_feature_run=no -nxt_feature_incs="${NXT_PHP_INCLUDE}" -nxt_feature_libs="${NXT_PHP_LIB} ${NXT_PHP_LDFLAGS}" -nxt_feature_test=" - #include - - int main(void) { - zend_async_event_t *e = NULL; - (void) e; - return 0; - }" - -. auto/feature - -if [ $nxt_found = yes ]; then - NXT_PHP_TRUEASYNC=1 - $echo " + PHP TrueAsync: enabled" -else - NXT_PHP_TRUEASYNC=0 - $echo " + PHP TrueAsync: not available (standard mode only)" -fi - - nxt_feature="PHP sapi_module_struct pre_request_init field" nxt_feature_name="" nxt_feature_run=no @@ -642,7 +617,6 @@ $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H \$(PP_CC) \$@ \$(v)\$(CC) -c \$(CFLAGS) $NXT_PHP_ADDITIONAL_FLAGS \$(NXT_INCS) \\ $NXT_PHP_INCLUDE -DNXT_ZEND_SIGNAL_STARTUP=$NXT_ZEND_SIGNAL_STARTUP \\ - -DNXT_PHP_TRUEASYNC=$NXT_PHP_TRUEASYNC \\ -DNXT_PHP_PRE_REQUEST_INIT=$NXT_PHP_PRE_REQUEST_INIT \\ -DNXT_PHP_MOUNTS_H=\"$NXT_PHP_MOUNTS_HEADER\" \\ $nxt_dep_flags \\ diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 4357a4bf7..e56e9cd8e 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -85,15 +85,6 @@ static nxt_int_t nxt_php_setup(nxt_task_t *task, nxt_process_t *process, nxt_common_app_conf_t *conf); static nxt_int_t nxt_php_start(nxt_task_t *task, nxt_process_data_t *data); static void nxt_php_cleanup_targets(void); -#if NXT_PHP_TRUEASYNC -static nxt_int_t nxt_php_async_load_entrypoint(nxt_task_t *task, nxt_str_t *entrypoint); -static bool nxt_php_activate_true_async(nxt_task_t *task); -static void nxt_php_suspend_coroutine(nxt_unit_ctx_t *ctx); -static int nxt_php_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); -static void nxt_php_remove_port(nxt_unit_t *unit, nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); -static void nxt_php_quit_handler(nxt_unit_ctx_t *ctx); -static void nxt_php_shm_ack_handler(nxt_unit_ctx_t *ctx); -#endif /* NXT_PHP_TRUEASYNC */ static nxt_int_t nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, nxt_conf_value_t *conf); static nxt_int_t nxt_php_set_ini_path(nxt_task_t *task, nxt_str_t *path, @@ -119,15 +110,8 @@ static nxt_int_t nxt_php_do_301(nxt_unit_request_info_t *req); static nxt_int_t nxt_php_handle_fs_err(nxt_unit_request_info_t *req); static void nxt_php_request_handler(nxt_unit_request_info_t *req); -#if NXT_PHP_TRUEASYNC -static void nxt_php_request_handler_async(nxt_unit_request_info_t *req); -#endif /* NXT_PHP_TRUEASYNC */ static void nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r); -#if NXT_PHP_TRUEASYNC -static void nxt_php_scope_init_superglobals(zend_async_scope_t *scope); -static void nxt_php_scope_populate_superglobals(zend_async_scope_t *scope); -#endif /* NXT_PHP_TRUEASYNC */ #if (PHP_VERSION_ID < 70400) static void nxt_zend_stream_init_fp(zend_file_handle *handle, FILE *fp, const char *filename); @@ -147,10 +131,6 @@ nxt_inline void nxt_php_set_str(nxt_unit_request_info_t *req, const char *name, static void nxt_php_set_cstr(nxt_unit_request_info_t *req, const char *name, const char *str, uint32_t len, zval *track_vars_array TSRMLS_DC); void nxt_php_register_variables(zval *track_vars_array TSRMLS_DC); -#if NXT_PHP_TRUEASYNC -static void nxt_php_register_variables_async(nxt_unit_request_info_t *req, - nxt_php_run_ctx_t *ctx, zval *track_vars_array TSRMLS_DC); -#endif /* NXT_PHP_TRUEASYNC */ #if NXT_PHP8 static void nxt_php_log_message(const char *message, int syslog_type_int); #else @@ -227,13 +207,6 @@ PHP_MINIT_FUNCTION(nxt_php_ext) nxt_php_chdir_handler = func->internal_function.handler; func->internal_function.handler = nxt_php_chdir; -#if NXT_PHP_TRUEASYNC - /* Register NginxUnit PHP extension classes (TrueAsync only) */ - if (nxt_php_extension_init() != NXT_OK) { - return FAILURE; - } -#endif /* NXT_PHP_TRUEASYNC */ - return SUCCESS; } @@ -397,8 +370,7 @@ static nxt_php_target_t *nxt_php_targets; static nxt_uint_t nxt_php_targets_count; static nxt_int_t nxt_php_last_target = -1; -/* Global Unit context - needed by nxt_php_extension.c */ -nxt_unit_ctx_t *nxt_php_unit_ctx; +static nxt_unit_ctx_t *nxt_php_unit_ctx; #if defined(ZTS) && (PHP_VERSION_ID < 70400) static void ***tsrm_ls; @@ -418,7 +390,6 @@ nxt_php_setup(nxt_task_t *task, nxt_process_t *process, static const nxt_str_t user_str = nxt_string("user"); static const nxt_str_t admin_str = nxt_string("admin"); - c = &conf->u.php; #ifdef ZTS @@ -489,205 +460,6 @@ nxt_php_setup(nxt_task_t *task, nxt_process_t *process, } -#if NXT_PHP_TRUEASYNC - -/** - * Add pending write to drain_queue - */ -nxt_php_pending_write_t * -nxt_php_drain_queue_add(nxt_unit_ctx_t *ctx, - nxt_unit_request_info_t *req, - zend_string *str, - size_t offset) -{ - nxt_php_async_ctx_data_t *async_data = ctx->data; - nxt_php_pending_write_t *pw; - - if (async_data == NULL) { - return NULL; - } - - pw = malloc(sizeof(nxt_php_pending_write_t)); - if (pw == NULL) { - return NULL; - } - - pw->req = req; - pw->offset = offset; - - /* Increase refcount - string won't be freed */ - pw->data = zend_string_copy(str); - - nxt_queue_insert_tail(&async_data->drain_queue, &pw->link); - - return pw; -} - - -/** - * Remove and free pending write from drain_queue - */ -static void -nxt_php_drain_queue_remove(nxt_php_pending_write_t *pw) -{ - nxt_queue_remove(&pw->link); - - /* Decrease refcount - will be freed when refcount reaches 0 */ - zend_string_release(pw->data); - - free(pw); -} - - -/** - * Callback function called when port fd becomes readable - */ -static void -nxt_php_port_event_callback(zend_async_event_t *event, zend_async_event_callback_t *callback, void *result, zend_object *exception) -{ - nxt_php_port_event_data_t *event_data; - - /* Get pointer to our extra data using extra_offset */ - event_data = (nxt_php_port_event_data_t *)((char *)event + event->extra_offset); - - /* Process messages from this port */ - nxt_unit_process_port_msg(event_data->ctx, event_data->port); -} - - -/** - * Callback to add port - called by Unit for each port - */ -static int -nxt_php_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) -{ - zend_async_poll_event_t *poll_event; - nxt_php_port_event_data_t *event_data; - - nxt_php_unit_ctx = ctx; - - /* Skip ports without input fd */ - if (port->in_fd == -1) { - return NXT_UNIT_OK; - } - - /* Set port to non-blocking mode */ - if (fcntl(port->in_fd, F_SETFL, O_NONBLOCK) == -1) { - nxt_unit_warn(ctx, "fcntl(%d, O_NONBLOCK) failed: %s (%d)", - port->in_fd, strerror(errno), errno); - return NXT_UNIT_ERROR; - } - - /* Create TrueAsync poll event with extra space for our data */ - poll_event = ZEND_ASYNC_NEW_POLL_EVENT_EX( - port->in_fd, false, ASYNC_READABLE, sizeof(nxt_php_port_event_data_t) - ); - - if (poll_event == NULL) { - nxt_unit_alert(ctx, "Failed to create TrueAsync poll event for port fd=%d", port->in_fd); - return NXT_UNIT_ERROR; - } - - /* Get pointer to our extra data */ - event_data = (nxt_php_port_event_data_t *)((char *)poll_event + poll_event->base.extra_offset); - event_data->ctx = ctx; - event_data->port = port; - - /* Register callback with poll event */ - if (!poll_event->base.add_callback(&poll_event->base, ZEND_ASYNC_EVENT_CALLBACK(nxt_php_port_event_callback))) { - nxt_unit_alert(ctx, "Failed to add callback for port fd=%d", port->in_fd); - poll_event->base.dispose(&poll_event->base); - return NXT_UNIT_ERROR; - } - - /* Start polling */ - if (!poll_event->base.start(&poll_event->base)) { - nxt_unit_alert(ctx, "Failed to start polling for port fd=%d", port->in_fd); - poll_event->base.dispose(&poll_event->base); - return NXT_UNIT_ERROR; - } - - /* Save poll_event in port->data for cleanup */ - port->data = poll_event; - - return NXT_UNIT_OK; -} - - -/** - * Callback to remove port - */ -static void -nxt_php_remove_port(nxt_unit_t *unit, nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) -{ - zend_async_poll_event_t *poll_event; - - if (port->data != NULL) { - poll_event = (zend_async_poll_event_t *) port->data; - - /* Dispose using TrueAsync cleanup */ - poll_event->base.dispose(&poll_event->base); - - port->data = NULL; - } -} - - -/** - * SHM ACK handler - called when Router releases shared memory - */ -static void -nxt_php_shm_ack_handler(nxt_unit_ctx_t *ctx) -{ - nxt_php_async_ctx_data_t *async_data = ctx->data; - nxt_queue_link_t *lnk; - nxt_php_pending_write_t *pw; - ssize_t res; - size_t remaining; - - if (async_data == NULL) { - return; - } - - /* Process all pending writes in drain_queue */ - lnk = nxt_queue_first(&async_data->drain_queue); - - while (lnk != nxt_queue_tail(&async_data->drain_queue)) { - pw = nxt_container_of(lnk, nxt_php_pending_write_t, link); - lnk = nxt_queue_next(lnk); /* Save next element before potential removal */ - - remaining = ZSTR_LEN(pw->data) - pw->offset; - - /* Try to send remaining data */ - res = nxt_unit_response_write_nb(pw->req, - ZSTR_VAL(pw->data) + pw->offset, - remaining, - 0); - - if (res < 0) { - /* Error - remove from queue */ - nxt_unit_warn(ctx, "drain_queue: write error, removing pending write"); - nxt_php_drain_queue_remove(pw); - continue; - } - - if (res == 0) { - /* Still no space - stop processing */ - return; - } - - pw->offset += res; - - if (pw->offset >= ZSTR_LEN(pw->data)) { - /* ALL SENT! Remove from queue */ - nxt_php_drain_queue_remove(pw); - } - } -} - -#endif /* NXT_PHP_TRUEASYNC */ - - static nxt_int_t nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) { @@ -701,7 +473,6 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) nxt_php_app_conf_t *c; nxt_common_app_conf_t *conf; - conf = data->app; c = &conf->u.php; @@ -742,36 +513,7 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) return ret; } - /* Choose request handler based on mode */ -#if NXT_PHP_TRUEASYNC - if (c->async && c->entrypoint.length > 0) { - nxt_debug(task, "PHP HTTP Server mode enabled with entrypoint: %V", &c->entrypoint); - php_init.callbacks.add_port = nxt_php_add_port; - php_init.callbacks.remove_port = nxt_php_remove_port; - php_init.callbacks.request_handler = nxt_php_request_handler_async; - php_init.callbacks.quit = nxt_php_quit_handler; - php_init.callbacks.shm_ack_handler = nxt_php_shm_ack_handler; - - if (nxt_slow_path(nxt_php_async_load_entrypoint(task, &c->entrypoint) != NXT_OK)) { - nxt_alert(task, "failed to load entrypoint script"); - return NXT_ERROR; - } - - if (nxt_php_request_callback == NULL) { - nxt_alert(task, "TrueAsync: Request callback not registered in the entrypoint script!"); - return NXT_ERROR; - } - - if(nxt_slow_path(!nxt_php_activate_true_async(task))) { - return NXT_ERROR; - } - } else { -#endif /* NXT_PHP_TRUEASYNC */ - nxt_debug(task, "PHP standard mode"); - php_init.callbacks.request_handler = nxt_php_request_handler; -#if NXT_PHP_TRUEASYNC - } -#endif /* NXT_PHP_TRUEASYNC */ + php_init.callbacks.request_handler = nxt_php_request_handler; unit_ctx = nxt_unit_init(&php_init); if (nxt_slow_path(unit_ctx == NULL)) { @@ -780,33 +522,7 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) nxt_php_unit_ctx = unit_ctx; -#if NXT_PHP_TRUEASYNC - /* Initialize async context data for drain_queue */ - if (c->async && c->entrypoint.length > 0) { - nxt_php_async_ctx_data_t *async_data; - - async_data = malloc(sizeof(nxt_php_async_ctx_data_t)); - if (async_data == NULL) { - nxt_alert(task, "Failed to allocate async context data"); - nxt_unit_done(unit_ctx); - return NXT_ERROR; - } - - nxt_queue_init(&async_data->drain_queue); - async_data->ctx = unit_ctx; - - unit_ctx->data = async_data; - } - - if (c->async && c->entrypoint.length > 0) { - /* Suspend main coroutine until the server continues to operate successfully */ - nxt_php_suspend_coroutine(nxt_php_unit_ctx); - } else { - nxt_unit_run(nxt_php_unit_ctx); - } -#else nxt_unit_run(nxt_php_unit_ctx); -#endif /* NXT_PHP_TRUEASYNC */ nxt_unit_done(nxt_php_unit_ctx); /* Clean up allocated memory before exit */ @@ -833,7 +549,6 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, static const nxt_str_t root_str = nxt_string("root"); static const nxt_str_t script_str = nxt_string("script"); static const nxt_str_t index_str = nxt_string("index"); - static const nxt_str_t entrypoint_str = nxt_string("entrypoint"); value = nxt_conf_get_object_member(conf, &root_str, NULL); @@ -915,74 +630,6 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, + target->root.length; } else { - /* Check for entrypoint (async mode) */ - value = nxt_conf_get_object_member(conf, &entrypoint_str, NULL); - - if (value != NULL) { - nxt_conf_get_string(value, &str); - - /* Check if entrypoint is an absolute path */ - if (str.length > 0 && str.start[0] == '/') { - /* Absolute path - use it directly */ - tmp = nxt_malloc(str.length + 1); - if (nxt_slow_path(tmp == NULL)) { - return NXT_ERROR; - } - - nxt_memcpy(tmp, str.start, str.length); - tmp[str.length] = '\0'; - - } else { - /* Relative path - prepend root */ - nxt_php_str_trim_lead(&str, '/'); - - tmp = nxt_malloc(target->root.length + 1 + str.length + 1); - if (nxt_slow_path(tmp == NULL)) { - return NXT_ERROR; - } - - p = tmp; - - p = nxt_cpymem(p, target->root.start, target->root.length); - *p++ = '/'; - - p = nxt_cpymem(p, str.start, str.length); - *p = '\0'; - } - - p = nxt_realpath(tmp); - if (nxt_slow_path(p == NULL)) { - nxt_alert(task, "entrypoint realpath(%s) failed %E", tmp, nxt_errno); - nxt_free(tmp); - return NXT_ERROR; - } - - nxt_free(tmp); - - target->script_filename.length = nxt_strlen(p); - target->script_filename.start = p; - - if (!nxt_str_start(&target->script_filename, - target->root.start, target->root.length)) - { - nxt_alert(task, "entrypoint is not under php root"); - nxt_free(p); - return NXT_ERROR; - } - - ret = nxt_php_dirname(&target->script_filename, - &target->script_dirname); - if (nxt_slow_path(ret != NXT_OK)) { - nxt_free(p); - return NXT_ERROR; - } - - target->script_name.length = target->script_filename.length - - target->root.length; - target->script_name.start = target->script_filename.start - + target->root.length; - } - value = nxt_conf_get_object_member(conf, &index_str, NULL); if (value != NULL) { @@ -1721,9 +1368,8 @@ nxt_php_unbuffered_write(const char *str, uint str_length TSRMLS_DC) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* Module startup, opcache.preload included, runs before any request. */ if (ctx == NULL || ctx->req == NULL) { - /* Log output from entrypoint to unit log */ nxt_unit_log(nxt_php_unit_ctx, NXT_UNIT_LOG_INFO, "PHP output: %.*s", (int)str_length, str); return str_length; @@ -1753,7 +1399,7 @@ nxt_php_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* Module startup, opcache.preload included, runs before any request. */ if (ctx == NULL || ctx->req == NULL) { return SAPI_HEADER_SENT_SUCCESSFULLY; } @@ -1834,7 +1480,10 @@ nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* + * fastcgi_finish_request() clears the request while PHP keeps running, + * and request_parse_body() can reach this callback afterwards. + */ if (ctx == NULL || ctx->req == NULL) { return 0; } @@ -1852,7 +1501,7 @@ nxt_php_read_cookies(TSRMLS_D) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* Module startup runs before any request. */ if (ctx == NULL) { return NULL; } @@ -1874,7 +1523,11 @@ nxt_php_register_variables(zval *track_vars_array TSRMLS_DC) ctx = SG(server_context); - /* During entrypoint execution there's no request context */ + /* + * PHP creates $_SERVER, and with it calls this callback, from script + * content rather than from request state, so it can be reached before + * the first request. + */ if (ctx == NULL || ctx->req == NULL) { return; } @@ -1986,130 +1639,6 @@ nxt_php_register_variables(zval *track_vars_array TSRMLS_DC) } -#if NXT_PHP_TRUEASYNC - -static void -nxt_php_register_variables_async(nxt_unit_request_info_t *req, - nxt_php_run_ctx_t *ctx, zval *track_vars_array TSRMLS_DC) -{ - const char *name; - char *str; - nxt_unit_field_t *f, *f_end; - nxt_unit_request_t *r; - - r = req->request; - - nxt_unit_req_debug(req, "nxt_php_register_variables_async"); - - /* Register SERVER_SOFTWARE */ - php_register_variable_safe((char *) "SERVER_SOFTWARE", - (char *) nxt_server.start, - nxt_server.length, track_vars_array TSRMLS_CC); - - /* Register SERVER_PROTOCOL */ - str = nxt_unit_sptr_get(&r->version); - php_register_variable_safe((char *) "SERVER_PROTOCOL", str, - r->version_length, track_vars_array TSRMLS_CC); - - /* Register PHP_SELF and PATH_INFO */ - if (ctx->path_info.length != 0) { - str = nxt_unit_sptr_get(&r->path); - php_register_variable_safe((char *) "PHP_SELF", str, - r->path_length, track_vars_array TSRMLS_CC); - php_register_variable_safe((char *) "PATH_INFO", - (char *) ctx->path_info.start, - ctx->path_info.length, track_vars_array TSRMLS_CC); - } else { - php_register_variable_safe((char *) "PHP_SELF", - (char *) ctx->script_name.start, - ctx->script_name.length, track_vars_array TSRMLS_CC); - } - - /* Register SCRIPT_NAME */ - php_register_variable_safe((char *) "SCRIPT_NAME", - (char *) ctx->script_name.start, - ctx->script_name.length, track_vars_array TSRMLS_CC); - - /* Register SCRIPT_FILENAME */ - php_register_variable_safe((char *) "SCRIPT_FILENAME", - (char *) ctx->script_filename.start, - ctx->script_filename.length, track_vars_array TSRMLS_CC); - - /* Register DOCUMENT_ROOT */ - php_register_variable_safe((char *) "DOCUMENT_ROOT", - (char *) ctx->root->start, - ctx->root->length, track_vars_array TSRMLS_CC); - - /* Register REQUEST_METHOD */ - str = nxt_unit_sptr_get(&r->method); - php_register_variable_safe((char *) "REQUEST_METHOD", str, - r->method_length, track_vars_array TSRMLS_CC); - - /* Register REQUEST_URI */ - str = nxt_unit_sptr_get(&r->target); - php_register_variable_safe((char *) "REQUEST_URI", str, - r->target_length, track_vars_array TSRMLS_CC); - - /* Register QUERY_STRING */ - str = nxt_unit_sptr_get(&r->query); - php_register_variable_safe((char *) "QUERY_STRING", str, - r->query_length, track_vars_array TSRMLS_CC); - - /* Register REMOTE_ADDR */ - str = nxt_unit_sptr_get(&r->remote); - php_register_variable_safe((char *) "REMOTE_ADDR", str, - r->remote_length, track_vars_array TSRMLS_CC); - - /* Register SERVER_ADDR */ - str = nxt_unit_sptr_get(&r->local_addr); - php_register_variable_safe((char *) "SERVER_ADDR", str, - r->local_addr_length, track_vars_array TSRMLS_CC); - - /* Register SERVER_NAME */ - str = nxt_unit_sptr_get(&r->server_name); - php_register_variable_safe((char *) "SERVER_NAME", str, - r->server_name_length, track_vars_array TSRMLS_CC); - - /* Register SERVER_PORT */ - str = nxt_unit_sptr_get(&r->local_port); - php_register_variable_safe((char *) "SERVER_PORT", str, - r->local_port_length, track_vars_array TSRMLS_CC); - - /* Register HTTPS if TLS is enabled */ - if (r->tls) { - php_register_variable_safe((char *) "HTTPS", (char *) "on", - 2, track_vars_array TSRMLS_CC); - } - - /* Register HTTP headers */ - f_end = r->fields + r->fields_count; - for (f = r->fields; f < f_end; f++) { - name = nxt_unit_sptr_get(&f->name); - str = nxt_unit_sptr_get(&f->value); - php_register_variable_safe((char *) name, str, - f->value_length, track_vars_array TSRMLS_CC); - } - - /* Register CONTENT_LENGTH */ - if (r->content_length_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->content_length_field; - str = nxt_unit_sptr_get(&f->value); - php_register_variable_safe((char *) "CONTENT_LENGTH", str, - f->value_length, track_vars_array TSRMLS_CC); - } - - /* Register CONTENT_TYPE */ - if (r->content_type_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->content_type_field; - str = nxt_unit_sptr_get(&f->value); - php_register_variable_safe((char *) "CONTENT_TYPE", str, - f->value_length, track_vars_array TSRMLS_CC); - } -} - -#endif /* NXT_PHP_TRUEASYNC */ - - static void nxt_php_set_sptr(nxt_unit_request_info_t *req, const char *name, nxt_unit_sptr_t *v, uint32_t len, zval *track_vars_array TSRMLS_DC) @@ -2210,6 +1739,7 @@ nxt_php_log_message(char *message TSRMLS_DC) ctx = SG(server_context); + /* fastcgi_finish_request() leaves the context without a request. */ if (ctx != NULL && ctx->req != NULL) { nxt_unit_req_log(ctx->req, NXT_UNIT_LOG_NOTICE, "php message: %s", message); @@ -2219,463 +1749,3 @@ nxt_php_log_message(char *message TSRMLS_DC) "php message: %s", message); } } - - -#if NXT_PHP_TRUEASYNC - -/* TrueAsync Mode Implementation */ - -/* External globals */ -extern zval *nxt_php_request_callback; - -/** - * The request handler invoked from NGINX UNIT. - * The handler uses a PHP callback function (nxt_php_request_callback) to process the request. - * - **/ -static void -nxt_php_request_handler_async(nxt_unit_request_info_t *req) -{ - zend_async_scope_t *request_scope; - zend_coroutine_t *coroutine; - nxt_php_run_ctx_t run_ctx; - nxt_php_target_t *target; - nxt_unit_request_t *r; - nxt_unit_field_t *f; - void *saved_server_context; - - r = req->request; - target = &nxt_php_targets[r->app_target]; - - /* 1. Create new Scope for this request with its own superglobals */ - request_scope = ZEND_ASYNC_NEW_SCOPE(ZEND_ASYNC_CURRENT_SCOPE); - - if (request_scope == NULL) { - nxt_unit_req_alert(req, "Failed to create request scope"); - nxt_unit_request_done(req, NXT_UNIT_ERROR); - return; - } - - /* 2. DISABLE inheriting global superglobals - each request has its own */ - ZEND_ASYNC_SCOPE_CLR_INHERIT_SUPERGLOBALS(request_scope); - - /* 3. Setup SG(server_context) and SG(request_info) for superglobals population */ - saved_server_context = SG(server_context); - - /* Prepare run context */ - nxt_memzero(&run_ctx, sizeof(run_ctx)); - run_ctx.req = req; - run_ctx.root = &target->root; - run_ctx.index = &target->index; - run_ctx.script_filename = target->script_filename; - run_ctx.script_dirname = target->script_dirname; - run_ctx.script_name = target->script_name; - - /* Extract cookie if present */ - if (r->cookie_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->cookie_field; - run_ctx.cookie = nxt_unit_sptr_get(&f->value); - } - - /* Setup SG(server_context) */ - SG(server_context) = &run_ctx; - - /* Setup SG(request_info) for php_default_treat_data */ - SG(request_info).request_method = nxt_unit_sptr_get(&r->method); - SG(request_info).query_string = r->query.offset ? nxt_unit_sptr_get(&r->query) : NULL; - SG(request_info).content_length = r->content_length; - - if (r->content_type_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->content_type_field; - SG(request_info).content_type = nxt_unit_sptr_get(&f->value); - } else { - SG(request_info).content_type = NULL; - } - - /* 4. Populate superglobals with data from HTTP request */ - nxt_php_scope_populate_superglobals(request_scope); - - /* 5. Restore original SG(server_context) */ - SG(server_context) = saved_server_context; - - /* 6. Create coroutine within this Scope */ - coroutine = ZEND_ASYNC_NEW_COROUTINE(request_scope); - if (coroutine == NULL) { - nxt_unit_req_alert(req, "Failed to create coroutine"); - /* Scope will be automatically cleaned up */ - nxt_unit_request_done(req, NXT_UNIT_ERROR); - return; - } - - /* 5. Setup coroutine entry point and data */ - coroutine->internal_entry = nxt_php_request_coroutine_entry; - coroutine->extended_data = req; - - /* 6. Enqueue coroutine for execution */ - if (!ZEND_ASYNC_ENQUEUE_COROUTINE(coroutine)) { - nxt_unit_req_alert(req, "Failed to enqueue coroutine"); - nxt_unit_request_done(req, NXT_UNIT_ERROR); - return; - } -} - -/* Load and execute entrypoint script in async mode */ -static nxt_int_t -nxt_php_async_load_entrypoint(nxt_task_t *task, nxt_str_t *entrypoint) -{ - FILE *fp; - const char *filename; - zend_file_handle file_handle; - - nxt_log(task, NXT_LOG_INFO, "TrueAsync: nxt_php_async_load_entrypoint called, PID=%d", getpid()); - - - /* Create null-terminated filename */ - char *filename_buf = nxt_malloc(entrypoint->length + 1); - if (filename_buf == NULL) { - nxt_alert(task, "TrueAsync: Failed to allocate filename buffer"); - return NXT_ERROR; - } - - nxt_memcpy(filename_buf, entrypoint->start, entrypoint->length); - filename_buf[entrypoint->length] = '\0'; - filename = filename_buf; - - - fp = fopen(filename, "re"); - if (fp == NULL) { - nxt_alert(task, "TrueAsync: Failed to open entrypoint %s", filename); - nxt_free(filename_buf); - return NXT_ERROR; - } - - /* Initialize minimal SG() fields like CLI does */ - SG(request_info).argc = 0; - SG(request_info).argv = NULL; - SG(request_info).path_translated = (char*)filename; - SG(server_context) = NULL; - - /* Call php_request_startup() like CLI does */ -#ifdef NXT_PHP7 - if (nxt_slow_path(php_request_startup() == FAILURE)) { -#else - if (nxt_slow_path(php_request_startup(TSRMLS_C) == FAILURE)) { -#endif - nxt_alert(task, "TrueAsync: php_request_startup() failed"); - fclose(fp); - nxt_free(filename_buf); - return NXT_ERROR; - } - - nxt_zend_stream_init_fp(&file_handle, fp, filename); - - /* Execute entrypoint script */ - int result = php_execute_script(&file_handle TSRMLS_CC); - - if (result == FAILURE) { - nxt_alert(task, "TrueAsync: php_execute_script() FAILED!"); - fclose(fp); - nxt_free(filename_buf); - return NXT_ERROR; - } - -#if (PHP_VERSION_ID >= 80100) - zend_destroy_file_handle(&file_handle); -#endif - - nxt_free(filename_buf); - - /* Check if callback was registered */ - if (nxt_php_request_callback == NULL) { - nxt_alert(task, "TrueAsync: No request handler registered! Call HttpServer->onRequest() in your entrypoint."); - return NXT_ERROR; - } - - /* Check callback is callable */ - if (nxt_php_request_callback != NULL && Z_TYPE_P(nxt_php_request_callback) == IS_OBJECT) { - nxt_log(task, NXT_LOG_INFO, "TrueAsync: Callback is object, type OK"); - } else { - nxt_alert(task, "TrueAsync: Callback has wrong type: %d", - nxt_php_request_callback ? Z_TYPE_P(nxt_php_request_callback) : -1); - } - - /* - * DON'T call php_request_shutdown() — we want the callback zval to - * persist across the prototype → worker fork. But scrub any - * pending exception left in EG: if the entrypoint script raised - * something that wasn't caught before HttpServer->onRequest() - * stored the callback, every forked worker would inherit the - * exception on its first request. The callback zval itself - * (nxt_php_request_callback) is not on the exception path; it - * was registered explicitly by the userland code. - * - * Other EG globals (output buffers, error_reporting, the symbol - * table) are also inherited, but those are reset implicitly when - * the worker enters a fresh request_init. Exception state is - * the one that bites hardest because the next request's - * php_execute_script() can early-exit on a stale EG(exception). - */ - if (EG(exception) != NULL) { - zend_clear_exception(); - } - - return NXT_OK; -} - - -/* Quit handler for graceful shutdown */ -static void -nxt_php_quit_handler(nxt_unit_ctx_t *ctx) -{ - nxt_unit_debug(ctx, "TrueAsync: quit handler called, triggering graceful shutdown"); - - /* Call ZEND_ASYNC_SHUTDOWN to trigger graceful shutdown of all coroutines */ - ZEND_ASYNC_SHUTDOWN(); -} - - -/* Server wait event methods */ -static bool -nxt_php_server_wait_event_start(zend_async_event_t *event) -{ - /* No action needed - event waits indefinitely */ - return true; -} - -static bool -nxt_php_server_wait_event_stop(zend_async_event_t *event) -{ - /* No action needed */ - return true; -} - -static bool -nxt_php_server_wait_event_add_callback(zend_async_event_t *event, zend_async_event_callback_t *callback) -{ - return zend_async_callbacks_push(event, callback); -} - -static bool -nxt_php_server_wait_event_del_callback(zend_async_event_t *event, zend_async_event_callback_t *callback) -{ - return zend_async_callbacks_remove(event, callback); -} - -static bool -nxt_php_server_wait_event_replay(zend_async_event_t *event, zend_async_event_callback_t *callback, zval *result, zend_object **exception) -{ - /* Event never resolves - cannot replay */ - return false; -} - -static zend_string * -nxt_php_server_wait_event_info(zend_async_event_t *event) -{ - return zend_string_init("NGINX Unit server waiting", sizeof("NGINX Unit server waiting") - 1, 0); -} - -static bool -nxt_php_server_wait_event_dispose(zend_async_event_t *event) -{ - if (ZEND_ASYNC_EVENT_REFCOUNT(event) > 1) { - ZEND_ASYNC_EVENT_DEL_REF(event); - return true; - } - - if (ZEND_ASYNC_EVENT_REFCOUNT(event) == 1) { - ZEND_ASYNC_EVENT_DEL_REF(event); - } - - /* Notify all callbacks that event is disposed */ - ZEND_ASYNC_CALLBACKS_NOTIFY(event, NULL, NULL); - - /* Free the event */ - efree(event); - - return true; -} - -static bool -nxt_php_activate_true_async(nxt_task_t *task) -{ - if (ZEND_ASYNC_IS_OFF) { - nxt_alert(task, "TrueAsync: async mode is off"); - return false; - } - - if (ZEND_ASYNC_IS_READY) { - if (!ZEND_ASYNC_SCHEDULER_LAUNCH()) { - nxt_alert(task, "TrueAsync: Failed to launch scheduler"); - return false; - } - } - - return ZEND_ASYNC_IS_ACTIVE; -} - -/** - * Suspend coroutine until application should finish - * Event handlers registered via add_port/remove_port will be triggered by scheduler - */ -static void -nxt_php_suspend_coroutine(nxt_unit_ctx_t *ctx) -{ - zend_coroutine_t *coroutine = ZEND_ASYNC_CURRENT_COROUTINE; - if (coroutine == NULL) { - nxt_unit_alert(ctx, "TrueAsync: Failed to get current coroutine"); - return; - } - - /* Create server wait event (it's custom nginx unit trigger for TrueAsync) */ - zend_async_event_t *event = emalloc(sizeof(zend_async_event_t)); - if (event == NULL) { - nxt_unit_alert(ctx, "TrueAsync: Failed to allocate server wait event"); - return; - } - - /* Initialize event structure */ - memset(event, 0, sizeof(zend_async_event_t)); - - /* Set event methods */ - event->start = nxt_php_server_wait_event_start; - event->stop = nxt_php_server_wait_event_stop; - event->add_callback = nxt_php_server_wait_event_add_callback; - event->del_callback = nxt_php_server_wait_event_del_callback; - event->replay = nxt_php_server_wait_event_replay; - event->info = nxt_php_server_wait_event_info; - event->dispose = nxt_php_server_wait_event_dispose; - - /* Create waker for coroutine */ - if (UNEXPECTED(zend_async_waker_new(coroutine) == NULL)) { - nxt_unit_alert(ctx, "TrueAsync: Failed to create waker"); - event->dispose(event); - return; - } - - /* Attach coroutine to wait event - it will suspend until GRACEFUL_SHUTDOWN */ - zend_async_resume_when(coroutine, event, true, zend_async_waker_callback_resolve, NULL); - - if (UNEXPECTED(EG(exception) != NULL)) { - nxt_unit_alert(ctx, "TrueAsync: Failed to attach coroutine to wait event"); - zend_async_waker_clean(coroutine); - event->dispose(event); - return; - } - - ZEND_ASYNC_SUSPEND(); - zend_async_waker_clean(coroutine); -} - - -static void -nxt_php_scope_init_superglobals(zend_async_scope_t *scope) -{ - zval tmp; - - if (scope->superglobals != NULL) { - return; - } - - ALLOC_HASHTABLE(scope->superglobals); - zend_hash_init(scope->superglobals, 8, NULL, ZVAL_PTR_DTOR, 0); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_GET", sizeof("_GET") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_POST", sizeof("_POST") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_COOKIE", sizeof("_COOKIE") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_SERVER", sizeof("_SERVER") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_ENV", sizeof("_ENV") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_FILES", sizeof("_FILES") - 1, &tmp); - - array_init(&tmp); - zend_hash_str_add_new(scope->superglobals, "_REQUEST", sizeof("_REQUEST") - 1, &tmp); -} - - -static void -nxt_php_scope_populate_superglobals(zend_async_scope_t *scope) -{ - zval *server_array, *get_array, *post_array, *cookie_array; - nxt_php_run_ctx_t *ctx; - nxt_unit_request_t *r; - nxt_unit_field_t *f; - - if (scope->superglobals == NULL) { - nxt_php_scope_init_superglobals(scope); - } - - ctx = SG(server_context); - if (ctx == NULL || ctx->req == NULL) { - return; - } - - r = ctx->req->request; - - server_array = zend_hash_str_find(scope->superglobals, "_SERVER", sizeof("_SERVER") - 1); - get_array = zend_hash_str_find(scope->superglobals, "_GET", sizeof("_GET") - 1); - post_array = zend_hash_str_find(scope->superglobals, "_POST", sizeof("_POST") - 1); - cookie_array = zend_hash_str_find(scope->superglobals, "_COOKIE", sizeof("_COOKIE") - 1); - - /* Populate $_SERVER */ - if (server_array != NULL) { - nxt_php_register_variables_async(ctx->req, ctx, server_array); - } - - /* Populate $_GET - use PARSE_STRING to write into our array */ - if (get_array != NULL && r->query_length > 0) { - char *query = estrndup((char *)nxt_unit_sptr_get(&r->query), r->query_length); - php_default_treat_data(PARSE_STRING, query, get_array); - } - - /* Populate $_COOKIE - use PARSE_STRING to write into our array */ - if (cookie_array != NULL && ctx->cookie != NULL) { - char *cookie = estrdup(ctx->cookie); - php_default_treat_data(PARSE_STRING, cookie, cookie_array); - } - - /* Populate $_POST - use PARSE_STRING for application/x-www-form-urlencoded */ - if (post_array != NULL && - SG(request_info).request_method && - !strcasecmp(SG(request_info).request_method, "POST")) - { - /* Check if content type is application/x-www-form-urlencoded */ - if (r->content_type_field != NXT_UNIT_NONE_FIELD) { - f = r->fields + r->content_type_field; - const char *content_type = nxt_unit_sptr_get(&f->value); - - /* Only parse if it's form-urlencoded (simple POST data) */ - if (content_type && - strncasecmp(content_type, "application/x-www-form-urlencoded", 33) == 0) - { - /* Read POST body into memory */ - size_t post_len = r->content_length; - if (post_len > 0) { - char *post_data = nxt_malloc(post_len + 1); - if (post_data != NULL) { - size_t read_bytes = nxt_unit_request_read(ctx->req, post_data, post_len); - if (read_bytes > 0) { - post_data[read_bytes] = '\0'; - /* php_default_treat_data will modify and free the string, so use estrndup */ - char *post_copy = estrndup(post_data, read_bytes); - php_default_treat_data(PARSE_STRING, post_copy, post_array); - } - nxt_free(post_data); - } - } - } - /* For multipart/form-data and others - not supported yet in async mode */ - } - } -} - -#endif /* NXT_PHP_TRUEASYNC */ diff --git a/test/php/async_mirror/entrypoint.php b/test/php/async_mirror/entrypoint.php deleted file mode 100644 index 81103115c..000000000 --- a/test/php/async_mirror/entrypoint.php +++ /dev/null @@ -1,20 +0,0 @@ -body(); - $request->respond(200, [ - 'Content-Type' => 'text/plain', - 'Content-Length' => (string) strlen($body), - ], $body); -}); diff --git a/test/php/async_shutdown/entrypoint.php b/test/php/async_shutdown/entrypoint.php deleted file mode 100644 index 3365720bb..000000000 --- a/test/php/async_shutdown/entrypoint.php +++ /dev/null @@ -1,22 +0,0 @@ -respond(200, ['Content-Type' => 'text/plain'], "OK\n"); -}); diff --git a/test/php/async_slow/entrypoint.php b/test/php/async_slow/entrypoint.php deleted file mode 100644 index 554e7f8e7..000000000 --- a/test/php/async_slow/entrypoint.php +++ /dev/null @@ -1,41 +0,0 @@ -query(), $params); - - $delay = min((int) ($params['sleep'] ?? 3), 30); - - // Write a sentinel file before sleeping so the test process can - // detect that this handler is genuinely in-flight (not just queued - // in the kernel TCP buffer). The path is passed as ?signal=. - // Writing the worker PID lets the test double-check which process is - // handling the request. - if (!empty($params['signal'])) { - file_put_contents($params['signal'], (string) getmypid()); - } - - // Async sleep: suspends this coroutine without blocking the event - // loop, allowing other coroutines (and the quit handler) to run. - \Async\sleep($delay); - - $request->respond(200, ['Content-Type' => 'text/plain'], "done\n"); -}); diff --git a/test/test_php_trueasync.py b/test/test_php_trueasync.py deleted file mode 100644 index 9d2e80ce8..000000000 --- a/test/test_php_trueasync.py +++ /dev/null @@ -1,564 +0,0 @@ -""" -Tests for PHP TrueAsync mode with graceful shutdown. - -Background ----------- -Commit 58ffa236 (cherry-picked from EdmondDantes/052afd8) adds -nxt_php_quit_handler() to src/nxt_php_sapi.c. When Unit sends a quit -signal to a PHP worker started in TrueAsync mode, the handler calls -ZEND_ASYNC_SHUTDOWN() so the coroutine scheduler terminates gracefully -instead of leaving the process hanging indefinitely. - -Status: TDD — tests are written for code that is not yet complete. -See TODO.md items #1 and #2. All tests in this file are automatically -skipped (via the require_trueasync autouse fixture) when: - - The "async"/"entrypoint" config fields are not parsed by Unit, OR - - nxt_php_extension.c is not built (no \\Unit\\Server PHP class). - -Prerequisites -------------- -- PHP 8.5+ (TrueAsync is a PHP 8.5+ feature). -- Unit PHP extension built from nxt_php_extension.c. -- The application config must set "async": true + "entrypoint" to - activate nxt_php_quit_handler registration in nxt_php_start(). - -Fixture layout --------------- - test/php/async_shutdown/entrypoint.php – minimal OK responder - test/php/async_mirror/entrypoint.php – echo request body - test/php/async_slow/entrypoint.php – sleeps N seconds; writes - sentinel file when handler starts - -Running -------- - sudo pytest-3 --print-log test/test_php_trueasync.py -""" - -import os -import re -import signal -import subprocess -import tempfile -import time - -import pytest - -from unit.applications.lang.php import ApplicationPHP -from unit.option import option - -# ────────────────────────────────────────────────────────────────────────────── -# Module-level prerequisites -# ────────────────────────────────────────────────────────────────────────────── - -# pytest_generate_tests() reads this and parametrises against each PHP -# version satisfying the filter — only 8.5+ gets TrueAsync support. -prerequisites = { - 'modules': { - 'php': lambda version: version.split('.')[:2] >= ['8', '5'], - } -} - -client = ApplicationPHP() - -# ────────────────────────────────────────────────────────────────────────────── -# Constants -# ────────────────────────────────────────────────────────────────────────────── - -# Seconds a worker is given to exit cleanly after the quit signal. -# Keep it generous on slow CI runners, but short enough to surface hangs. -SHUTDOWN_TIMEOUT = 15 - -# Seconds to poll before declaring a worker "failed to start". -WORKER_START_TIMEOUT = 10 - -# ────────────────────────────────────────────────────────────────────────────── -# Module-level autouse fixture — replaces per-test runtime probes -# ────────────────────────────────────────────────────────────────────────────── - -@pytest.fixture(autouse=True) -def require_trueasync(): - """ - Skip every test in this module if the TrueAsync infrastructure is absent. - - Runs before each test body (after the conftest 'run' fixture starts Unit). - Three distinct failure modes are distinguished so the skip message is - actionable and unambiguous: - - • PHP version < 8.5 → TrueAsync C code was compiled out via - #if NXT_PHP_TRUEASYNC (which configure sets to 0 when zend_async_event_t - is absent from the PHP headers). The skip message points to the version - guard, not to a missing extension. This catches any edge case where an - older PHP version slips past the prerequisites filter. - - • Config rejected with "error" → "async"/"entrypoint" fields are missing - from nxt_php_app_conf_t (TODO.md #1). The fix is in nxt_application.h. - - • Config accepted but status != 200 → nxt_php_extension.c is not built, - so the PHP worker starts but \\Unit\\Server is undefined and the entrypoint - fails to register a request callback (TODO.md #2). - - The probe config is intentionally minimal (async_shutdown fixture, one - process) to minimise side-effects. The test body is responsible for - loading whatever config it actually needs — client.conf() in the test body - will replace the probe config before any real assertions run. - """ - # ── Guard 1: PHP version ──────────────────────────────────────────────── - # TrueAsync C code is compiled only when configure detects - # zend_async_event_t in the PHP headers (NXT_PHP_TRUEASYNC=1), which - # requires PHP 8.5+. A misleading "extension not available" message - # would appear on older PHP if we went straight to the HTTP probe. - php_type = client.get_application_type() # e.g. "php 8.5.0" - match = re.search(r'(\d+)\.(\d+)', php_type) - if not match or [match.group(1), match.group(2)] < ['8', '5']: - pytest.skip( - f'TrueAsync requires PHP 8.5+ with zend_async_event_t in headers ' - f'(got {php_type!r}); C code excluded by #if NXT_PHP_TRUEASYNC' - ) - - # ── Guard 2 & 3: runtime probe ────────────────────────────────────────── - probe_conf = { - 'listeners': {'*:8080': {'pass': 'applications/_trueasync_probe'}}, - 'applications': { - '_trueasync_probe': { - 'type': client.get_application_type(), - 'processes': {'spare': 0}, - 'root': f'{option.test_dir}/php/async_shutdown', - 'async': True, - 'entrypoint': 'entrypoint.php', - } - }, - } - - r = client.conf(probe_conf) - if 'error' in r: - pytest.skip( - 'PHP async config rejected — "async"/"entrypoint" fields are ' - 'missing from nxt_php_app_conf_t (see TODO.md #1)' - ) - - if client.get()['status'] != 200: - pytest.skip( - 'PHP TrueAsync extension not available — ' - 'build nxt_php_extension.c and relink the PHP module ' - '(see TODO.md #2)' - ) - - yield # test body runs here; probe config is replaced by the test itself - - -# ────────────────────────────────────────────────────────────────────────────── -# Helpers -# ────────────────────────────────────────────────────────────────────────────── - -def _async_app_conf(script: str) -> dict: - """ - Return a Unit application config dict for TrueAsync mode. - - "async": true + "entrypoint" is the combination that activates - nxt_php_quit_handler registration inside nxt_php_start(). - """ - return { - 'type': client.get_application_type(), - 'processes': {'spare': 0}, - 'root': f'{option.test_dir}/php/{script}', - 'async': True, - 'entrypoint': 'entrypoint.php', - } - - -def _worker_pids(app_name: str, unit_pid: int) -> list: - """ - Return PIDs of live worker processes for *app_name*. - - Unit names application workers after the application name, so we - match the Unit master PID as ppid and the app name in the command. - """ - output = subprocess.check_output(['ps', 'ax', '-O', 'ppid']).decode() - return re.findall( - fr'\s*(\d+)\s+{unit_pid}\s+.*{re.escape(app_name)}', - output, - ) - - -def _wait_for_worker(app_name: str, unit_pid: int, - timeout: float = WORKER_START_TIMEOUT) -> list: - """ - Poll until at least one worker for *app_name* appears in /proc. - Raises AssertionError if no worker appears within *timeout* seconds. - """ - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - pids = _worker_pids(app_name, unit_pid) - if pids: - return pids - time.sleep(0.1) - raise AssertionError( - f'Worker for {app_name!r} did not start within {timeout}s' - ) - - -def _workers_gone(pids: list, timeout: float = SHUTDOWN_TIMEOUT) -> bool: - """ - Return True once every PID in *pids* has vanished from /proc. - Return False if any remain alive after *timeout* seconds. - - Checking /proc/{pid} is more reliable than parsing 'ps' output: - the entry disappears exactly when the kernel reaps the process, - with no race between the read and the process exiting. - """ - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if all(not os.path.exists(f'/proc/{p}') for p in pids): - return True - time.sleep(0.1) - return False - - -def _wait_for_file(path: str, timeout: float = WORKER_START_TIMEOUT) -> bool: - """ - Poll until *path* exists on disk (written by the PHP handler as a - start sentinel) or *timeout* elapses. Return True if found in time. - """ - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if os.path.exists(path): - return True - time.sleep(0.05) - return False - - -# ────────────────────────────────────────────────────────────────────────────── -# Tests -# ────────────────────────────────────────────────────────────────────────────── - - -def test_php_trueasync_worker_starts(unit_pid): - """ - Smoke test: a PHP worker in TrueAsync mode starts and serves requests. - - This is the minimal sanity check for every other test. If it fails - after the require_trueasync fixture passed, the most likely cause is - a config field being silently ignored rather than raising an error - (e.g. "async" accepted but not wired to c->async in nxt_php_start). - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_shutdown'}}, - 'applications': {'async_shutdown': _async_app_conf('async_shutdown')}, - } - ), 'configuration accepted' - - pids = _wait_for_worker('async_shutdown', unit_pid) - assert pids, 'at least one worker process is running' - - resp = client.get() - assert resp['status'] == 200, 'worker responds with 200' - assert 'OK' in resp['body'], 'response body from async handler' - - -def test_php_trueasync_graceful_shutdown_on_reconfigure(unit_pid, wait_for_record): - """ - Core regression test for commit 58ffa236. - - Scenario - -------- - 1. Start a PHP worker in TrueAsync mode. - 2. Confirm it is alive and serving. - 3. Remove the application — Unit's router sends the quit signal to - the worker, exercising nxt_php_quit_handler() → ZEND_ASYNC_SHUTDOWN(). - 4. Assert the worker exits within SHUTDOWN_TIMEOUT seconds. - - Before the fix the worker blocked in nxt_unit_run() indefinitely - because the scheduler never received a shutdown trigger. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_app'}}, - 'applications': {'async_app': _async_app_conf('async_shutdown')}, - } - ), 'load async_shutdown configuration' - - # Capture PIDs before reconfiguration so we can track these specific - # processes — not just any worker that might appear later. - pids = _wait_for_worker('async_app', unit_pid) - - assert client.get()['status'] == 200, 'worker alive before reconfigure' - - # Remove the application; Unit will deliver the quit signal. - assert 'success' in client.conf( - {'listeners': {}, 'applications': {}} - ), 'empty configuration accepted' - - assert _workers_gone(pids), ( - f'TrueAsync worker(s) {pids} did not exit within {SHUTDOWN_TIMEOUT}s ' - f'— nxt_php_quit_handler may not have fired, or ' - f'ZEND_ASYNC_SHUTDOWN() did not stop the scheduler' - ) - - # Informational: verify Unit logged the quit handler invocation. - # Only present when Unit is built with debug logging; not a hard failure. - wait_for_record(r'TrueAsync: quit handler called', wait=5) - - -def test_php_trueasync_graceful_shutdown_on_sigquit(unit_pid, skip_alert): - """ - Graceful shutdown when SIGQUIT is delivered to the Unit master. - - This is the normal production shutdown path: systemd ExecStop, - container runtime "stop", or NGINX-style graceful reload all use SIGQUIT. - The signal path is: SIGQUIT → master → router → quit callback → ZEND_ASYNC_SHUTDOWN. - - skip_alert suppresses the "exited on signal" log alert that would - otherwise cause the test-teardown checker to flag a failure. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_app'}}, - 'applications': {'async_app': _async_app_conf('async_shutdown')}, - } - ), 'load configuration' - - pids = _wait_for_worker('async_app', unit_pid) - - assert client.get()['status'] == 200, 'worker alive' - - for pid in pids: - skip_alert(fr'process {pid} exited on signal') - - os.kill(unit_pid, signal.SIGQUIT) - - assert _workers_gone(pids), ( - f'TrueAsync worker(s) did not exit within {SHUTDOWN_TIMEOUT}s ' - f'after SIGQUIT → Unit master (pid={unit_pid})' - ) - - -def test_php_trueasync_no_zombie_after_shutdown(unit_pid, skip_alert): - """ - Exited TrueAsync workers must be reaped — no zombies. - - A zombie means the master is not calling waitpid() after the quit - handler returns. This is a process-management regression independent - of TrueAsync, but worth asserting here alongside the shutdown tests. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_app'}}, - 'applications': {'async_app': _async_app_conf('async_shutdown')}, - } - ), 'load configuration' - - pids = _wait_for_worker('async_app', unit_pid) - for pid in pids: - skip_alert(fr'process {pid} exited on signal') - - assert 'success' in client.conf( - {'listeners': {}, 'applications': {}} - ), 'empty configuration to trigger quit' - - assert _workers_gone(pids), 'workers exited before zombie check' - - out = subprocess.check_output( - ['ps', 'ax', '-o', 'state', '-o', 'ppid'] - ).decode() - zombie_ppids = re.findall(r'Z\s+(\d+)', out) - assert str(unit_pid) not in zombie_ppids, ( - 'No zombie processes parented to Unit master after async worker shutdown' - ) - - -def test_php_trueasync_standard_mode_unaffected(): - """ - Regression guard: nxt_php_sapi.c changes must not break standard PHP. - - Standard mode uses nxt_php_request_handler (not the async variant). - nxt_php_quit_handler is NOT registered (php_init.callbacks.quit = NULL) - and the pre-existing shutdown path is used unchanged. - """ - # Load the plain mirror fixture — no async config involved. - client.load('mirror') - - resp = client.post(body='hello', headers={'Content-Length': '5'}) - assert resp['status'] == 200, 'standard mode: POST succeeds' - assert resp['body'] == 'hello', 'standard mode: body echoed' - - assert client.get()['status'] == 200, 'standard mode: GET succeeds' - - -def test_php_trueasync_mirror_request_body(unit_pid): - """ - Functional test: async handler reads and echoes the request body. - - Exercises nxt_php_request_handler_async(), - nxt_php_scope_populate_superglobals(), and the response-write path - including nxt_php_drain_queue under backpressure. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_mirror'}}, - 'applications': {'async_mirror': _async_app_conf('async_mirror')}, - } - ), 'load async_mirror configuration' - - _wait_for_worker('async_mirror', unit_pid) - - payload = 'TrueAsync echo test payload' - resp = client.post( - body=payload, - headers={'Content-Length': str(len(payload))}, - ) - assert resp['status'] == 200, 'async mirror: status 200' - assert resp['body'] == payload, 'async mirror: body echoed correctly' - - -def test_php_trueasync_multiple_workers_all_exit(unit_pid, skip_alert): - """ - All worker processes must exit gracefully on shutdown, not just one. - - Each worker runs nxt_php_start() independently, so nxt_php_quit_handler - must be registered per-worker (not once globally). ZEND_ASYNC_SHUTDOWN() - must stop each worker's own scheduler instance. - """ - conf = _async_app_conf('async_shutdown') - conf['processes'] = 2 # request two workers explicitly - - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_multi'}}, - 'applications': {'async_multi': conf}, - } - ), 'load multi-worker configuration' - - # Wait until both workers are visible in the process table. - deadline = time.monotonic() + WORKER_START_TIMEOUT - pids = [] - while time.monotonic() < deadline: - pids = _worker_pids('async_multi', unit_pid) - if len(pids) >= 2: - break - time.sleep(0.1) - - assert len(pids) >= 2, ( - f'Expected ≥2 TrueAsync workers, got {len(pids)}: {pids}' - ) - - for pid in pids: - skip_alert(fr'process {pid} exited on signal') - - assert 'success' in client.conf( - {'listeners': {}, 'applications': {}} - ), 'empty config to stop all workers' - - still_alive = [p for p in pids if os.path.exists(f'/proc/{p}')] - assert _workers_gone(pids), ( - f'Not all TrueAsync workers exited within {SHUTDOWN_TIMEOUT}s. ' - f'Still alive: {still_alive}' - ) - - -def test_php_trueasync_inflight_request_completes(unit_pid, skip_alert, tmp_path): - """ - In-flight requests must complete before the worker exits on shutdown. - - Approach - -------- - 1. Start the slow entrypoint (?sleep=2). The handler writes a sentinel - file to *signal_file* the moment it begins executing — before sleeping. - 2. Poll *signal_file* instead of using time.sleep(): once the file exists - the handler is definitively in-flight inside the PHP coroutine. - 3. Issue the reconfigure (removes the app, triggers the quit signal). - 4. Wait for curl to finish and assert the response is 200 "done". - - The slow request is made via curl in a subprocess rather than a thread: - subprocess.Popen avoids GIL contention and gives an independent TCP - connection not shared with the client used for conf() calls. - - NOTE: This test asserts cooperative-shutdown semantics — that - ZEND_ASYNC_SHUTDOWN() lets active coroutines run to completion. - If the TrueAsync scheduler cancels coroutines on shutdown instead, - this test will fail with a connection error or non-200 status; see - TODO.md #4 for the discussion and resolution path. - """ - assert 'success' in client.conf( - { - 'listeners': {'*:8080': {'pass': 'applications/async_slow'}}, - 'applications': {'async_slow': _async_app_conf('async_slow')}, - } - ), 'load async_slow configuration' - - pids = _wait_for_worker('async_slow', unit_pid) - for pid in pids: - skip_alert(fr'process {pid} exited on signal') - - # Build the sentinel file path in /tmp rather than in pytest's tmp_path. - # - # tmp_path is owned by the user running pytest (typically root in CI), - # but the Unit PHP worker runs under its own user (e.g. "unit"). The - # worker process would get EACCES writing to a 0o700 directory it does - # not own. /tmp is world-writable (mode 1777) on every POSIX system, - # so no chmod dance is needed. - # - # We include the pytest tmp_path basename (unique per-test) to avoid - # collisions when multiple test runs overlap, and clean up explicitly - # at the end of the test. - signal_file = os.path.join( - tempfile.gettempdir(), - f'unit_async_{os.path.basename(str(tmp_path))}', - ) - - # Launch curl in a subprocess: one independent TCP connection, - # no GIL contention, clean resource lifetime via context manager. - url = f'http://localhost:8080/?sleep=2&signal={signal_file}' - with subprocess.Popen( - ['curl', '-s', '--max-time', '25', url], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) as proc: - - # Wait for the PHP handler to create the sentinel file. - # This is the authoritative signal that the coroutine is sleeping - # inside the handler — not just enqueued in the kernel or router. - if not _wait_for_file(signal_file): - proc.terminate() - pytest.fail( - f'Slow request did not reach the PHP handler within ' - f'{WORKER_START_TIMEOUT}s (sentinel file not created). ' - f'Check that async_slow/entrypoint.php writes ?signal= path.' - ) - - # The request is now definitively in-flight. Trigger shutdown. - assert 'success' in client.conf( - {'listeners': {}, 'applications': {}} - ), 'reconfigure while request is in flight' - - # Wait for curl to finish; generous timeout covers sleep=2 plus - # scheduler shutdown overhead. - try: - stdout, _ = proc.communicate(timeout=30) - except subprocess.TimeoutExpired: - proc.kill() - pytest.fail('curl did not finish within 30s after shutdown trigger') - - body = stdout.decode() - - assert proc.returncode == 0, ( - f'curl exited with code {proc.returncode} — connection was dropped ' - f'before the response arrived. ZEND_ASYNC_SHUTDOWN() may cancel ' - f'in-flight coroutines rather than letting them complete (TODO.md #4).' - ) - assert 'done' in body, ( - f'Unexpected response body: {body!r}. ' - f'Expected "done" from async_slow handler.' - ) - - assert _workers_gone(pids), ( - f'Worker(s) did not exit within {SHUTDOWN_TIMEOUT}s after the ' - f'in-flight request completed.' - ) - - # Remove the sentinel file written by the PHP worker. It lives in /tmp - # (not in pytest's tmp_path) so pytest cannot clean it up automatically. - try: - os.unlink(signal_file) - except FileNotFoundError: - pass # handler may not have written it if the test failed early From 118b28f31fc411a98655ea772c0f6eb977408e6b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 17 Sep 2026 20:17:41 +0200 Subject: [PATCH 214/391] changelog: the PHP TrueAsync code path is removed Both files, same text, appended to the end of the unreleased 1.36.2 stanza. The 1.35.2 and 1.35.3 entries that describe the TrueAsync work are left as they were released. Co-Authored-By: DeepSeek V4.1 Flash Harness: DeepSeek Harness --- CHANGES | 10 ++++++++++ docs/changes.xml | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGES b/CHANGES index 9b1d89e8a..ffc109500 100644 --- a/CHANGES +++ b/CHANGES @@ -345,6 +345,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 The OpenSSL backend is unaffected; none of these files were ever part of a default build. + *) Change: removed the PHP TrueAsync code path. It was written for a + "zend_async_event_t" API expected in PHP 8.5 core, which did not ship + it, so the configure probe never succeeded and the code sat behind + "#if NXT_PHP_TRUEASYNC" in every build. It could not have compiled had + the probe succeeded: it reads "async" and "entrypoint" members that + "nxt_php_app_conf_t" does not have and calls "nxt_php_extension_init()", + which no file in the tree defines. The probe, the "entrypoint" target + key and the TrueAsync test suite are removed with it; a PHP application + builds and behaves as before. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index bc2091025..33ce15483 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -456,6 +456,19 @@ that only the refusal path still reached, and two crates with it. + + +removed the PHP TrueAsync code path. It was written for a "zend_async_event_t" +API expected in PHP 8.5 core, which did not ship it, so the configure probe +never succeeded and the code sat behind "#if NXT_PHP_TRUEASYNC" in every build. +It could not have compiled had the probe succeeded: it reads "async" and +"entrypoint" members that "nxt_php_app_conf_t" does not have and calls +"nxt_php_extension_init()", which no file in the tree defines. The probe, the +"entrypoint" target key and the TrueAsync test suite are removed with it; a PHP +application builds and behaves as before. + + + From 3a223d91783527571c2d5ca311b97a4ee74e12b0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 07:23:18 +0200 Subject: [PATCH 215/391] router: count a worker that answered and kept running An application can answer a request and keep executing. PHP does it whenever a script calls fastcgi_finish_request(): libunit reports the request done, the last response message reaches the router, and the script runs on. The router took that message at face value. nxt_router_app_port_release() sees no active request, moves the port to idle_ports and stamps idle_start, so a worker in the middle of a script is counted idle, its slot goes back to "processes": {"max"}, and at idle_start + idle_timeout the reaper deregisters it and sends a QUIT it cannot read until the script returns. The process is still running the whole time. Only the worker knows, so it now says so. NXT_PORT_MSG_DETACHED carries one byte: the application answered and is still running, or that work has finished. A port in the first state is skipped by the idle transition -- the same way active_websockets already skips it -- so it stays in app->ports and in app->processes, never enters idle_ports, and is therefore invisible to the reaper and still counted against "max". The finish edge runs that transition, which is why it was extracted into nxt_router_app_port_idle(). A new message type rather than a flag on nxt_port_msg_t: that header has no spare bit that is reliably zeroed. Its four "1 bit" fields are whole bytes, its trailing pad byte is never cleared by the senders that build the header field by field, and nxt_port_socket_write() ORs into ->last. A type is bounds-checked against NXT_PORT_MSG_MAX on both sides, so a peer that predates it refuses the message instead of misreading a flag. The slot is appended, which is the only index-stable edit to nxt_port_handlers_t: every _NXT_PORT_MSG_* value is that struct's offset. Both edges go to the router's own port, the way OOSM does, not to the port the response went to. That port is read on the router's main thread, which is where everything the edge touches lives: rt->ports is read without a lock, and an application is freed on app->engine, the same thread. One socket per worker also keeps a finish edge behind the start edge that preceded it; the next request's response may belong to another router engine, and two engines' reads of two sockets race, so an edge per response port would let the first request's finish clear the flag under the second request's work. The router applies an edge only for the process that sent it. The pid in a port message is chosen by the sender, and every worker holds a copy of the router port, so a worker could otherwise pin another worker as detached, with a reference to its application, or release one to the reaper. The router port has no shared memory queue on either side, so the edge always arrives on the socket with the kernel's credentials, and nxt_recv_msg_cmsg_pid() must equal port_msg.pid. This also holds for a worker in its own pid namespace: the credential is translated to the router's namespace, and port_msg.pid is the global pid the router gave the worker. src/test/nxt_router_detached_test.c sends forged and genuine edges; without the check, a forged start sets the flag. The start edge is still sent before the last response, but the two go to different sockets, so nothing orders them on the router's side. A start edge that finds its worker already parked as idle -- because the response was read first, or because the request was failed rather than answered and released the worker with no edge in sight -- takes it back out, the way an acknowledgement does; nxt_router_app_port_busy() is that unwind, extracted from the acknowledgement handler so the flag and the unwind happen in one critical section. Three things a detached worker needs that an idle one does not: - An application reference. It holds no request, and a request is what otherwise holds the application: nxt_request_rpc_data_unlink() has already dropped its reference by then, so a configuration reload could free the application out from under a running process. Taken with an increment-unless-zero, because the last drop can happen on a worker engine while the edge is already on its way, and nxt_router_free_app() runs unconditionally once posted: a plain increment would resurrect an application whose free is queued behind the handler, and the start it may post would lock a mutex in freed memory. Zero is treated like no application at all. Kept only by a start edge that changed the state, so a repeated start cannot leak one. - Cleanup when it dies mid-work. A fatal error after the response, or a kill, reaches only nxt_router_app_port_close(), which now settles the count there. nxt_router_free_app() asserts it reached zero. - Protection from a graceful QUIT. libunit's gate tested active_req, which a detached worker satisfies; it would have torn down the read port with the script still running. A QUIT deferred that way is retried when the work ends, as one deferred on a request is retried when the request is released. libunit reports the finish edge itself, from all three sites that call the application's request handler, so an application does not have to pair the call with anything and every return path is covered. "/status" gains a "detached" count beside running, starting and idle. Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Fable 5.1 --- auto/sources | 1 + docs/unit-openapi.yaml | 45 +++ src/nxt_port.h | 41 +++ src/nxt_router.c | 466 +++++++++++++++++++++++----- src/nxt_router.h | 15 + src/nxt_status.c | 5 +- src/nxt_status.h | 1 + src/nxt_unit.c | 147 ++++++++- src/nxt_unit.h | 19 ++ src/test/nxt_router_detached_test.c | 223 +++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 12 files changed, 896 insertions(+), 72 deletions(-) create mode 100644 src/test/nxt_router_detached_test.c diff --git a/auto/sources b/auto/sources index 6d5c873a5..f63350349 100644 --- a/auto/sources +++ b/auto/sources @@ -178,6 +178,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_proto_death_test.c \ src/test/nxt_router_start_timeout_test.c \ src/test/nxt_router_remove_pid_soak_test.c \ + src/test/nxt_router_detached_test.c \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ src/test/nxt_proto_creating_wedge_test.c \ diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 065b51e39..0a1a20123 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -5478,6 +5478,39 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /status/applications/{appName}/processes/detached: + summary: "Endpoint for the `detached` processes number" + get: + operationId: getStatusApplicationsAppProcessesDetached + summary: "Retrieve the detached processes app status number" + description: "Retrieves the `detached` processes number that + represents Unit's per-app + [process statistics](https://unit.nginx.org/statusapi/)." + + tags: + - status + + parameters: + - $ref: "#/components/parameters/appName" + + responses: + "200": + description: "OK; the `detached` number exists in the + configuration." + + content: + application/json: + schema: + type: integer + format: int64 + + examples: + Detached: + value: 0 + + "404": + $ref: "#/components/responses/responseNotFound" + /status/applications/{appName}/processes/idle: summary: "Endpoint for the `idle` processes number" get: @@ -6344,6 +6377,7 @@ components: starting: 1 unaccounted: 0 idle: 0 + detached: 0 requests: active: 15 @@ -6405,6 +6439,7 @@ components: starting: 1 unaccounted: 0 idle: 0 + detached: 0 requests: active: 15 @@ -6417,6 +6452,7 @@ components: starting: 1 unaccounted: 0 idle: 0 + detached: 0 requests: active: 15 @@ -6428,6 +6464,7 @@ components: starting: 1 unaccounted: 0 idle: 0 + detached: 0 # /status/applications/{appName}/requests statusApplicationsAppRequests: @@ -7986,6 +8023,14 @@ components: They count against `processes/max` until they announce themselves after all, or exit." + detached: + type: integer + format: int64 + description: "App processes that answered a request and kept + running, such as a PHP script after `fastcgi_finish_request()`. + They are counted in `running`, never in `idle`, and hold their + slot under `processes/max` until the work ends." + idle: type: integer description: "Current idle app processes." diff --git a/src/nxt_port.h b/src/nxt_port.h index b2db9bb39..bf233b543 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -63,6 +63,14 @@ struct nxt_port_handlers_s { nxt_port_handler_t shm_ack; nxt_port_handler_t read_queue; nxt_port_handler_t read_socket; + + /* + * An application that answered a request and kept running, and later + * that it has finished. Appended, and appended is the only safe edit + * here: every _NXT_PORT_MSG_* value is this struct's offset, so + * inserting or reordering a slot renumbers the wire protocol. + */ + nxt_port_handler_t detached; }; @@ -120,6 +128,8 @@ typedef enum { _NXT_PORT_MSG_READ_QUEUE = nxt_port_handler_idx(read_queue), _NXT_PORT_MSG_READ_SOCKET = nxt_port_handler_idx(read_socket), + _NXT_PORT_MSG_DETACHED = nxt_port_handler_idx(detached), + NXT_PORT_MSG_MAX = sizeof(nxt_port_handlers_t) / sizeof(nxt_port_handler_t), @@ -164,6 +174,7 @@ typedef enum { NXT_PORT_MSG_SHM_ACK = nxt_msg_last(_NXT_PORT_MSG_SHM_ACK), NXT_PORT_MSG_READ_QUEUE = _NXT_PORT_MSG_READ_QUEUE, NXT_PORT_MSG_READ_SOCKET = _NXT_PORT_MSG_READ_SOCKET, + NXT_PORT_MSG_DETACHED = nxt_msg_last(_NXT_PORT_MSG_DETACHED), } nxt_port_msg_type_t; @@ -179,6 +190,24 @@ typedef enum { } nxt_port_quit_mode_t; +/* + * Wire-format payload for NXT_PORT_MSG_DETACHED. A single byte says which + * edge this is: an application that has answered a request and is still + * running, or the same application reporting that work done. + * + * A byte rather than a flag on nxt_port_msg_t: that header has no spare + * bit that is reliably zeroed. Its four "1 bit" fields are whole bytes, + * the trailing pad byte is never cleared by the senders that build the + * header field by field, and nxt_port_socket_write() ORs into ->last. A + * new type is bounds-checked on both sides instead, so an older peer + * refuses the message rather than misreading a flag. + */ +typedef enum { + NXT_PORT_DETACHED_START = 0, + NXT_PORT_DETACHED_FINISH = 1, +} nxt_port_detached_t; + + /* Passed as a first iov chunk. */ typedef struct { uint32_t stream; @@ -306,6 +335,18 @@ struct nxt_port_s { uint32_t max_share; uint32_t active_websockets; + + /* + * The application answered a request on this port and kept running. + * Treated exactly like active_websockets by the idle transition in + * nxt_router_app_port_release(): the port stays in app->ports and in + * app->processes, and stays out of the idle queues, so the reaper + * never sees it and it keeps counting against "processes": {"max"}. + * + * Unlike active_websockets this one is cleared again, when the + * application reports the work finished. + */ + uint8_t detached; uint32_t active_requests; nxt_port_handler_t handler; diff --git a/src/nxt_router.c b/src/nxt_router.c index 70ec73c0b..4d1efef26 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -402,6 +402,14 @@ static void nxt_router_http_request_release_post(nxt_task_t *task, static void nxt_router_http_request_release(nxt_task_t *task, void *obj, void *data); static void nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +static void nxt_router_detached_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, + uint8_t state); +static nxt_bool_t nxt_router_app_port_idle(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port); +static nxt_bool_t nxt_router_app_port_busy(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port, const char *reason); static void nxt_router_get_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_router_get_mmap_handler(nxt_task_t *task, @@ -441,6 +449,7 @@ static const nxt_port_handlers_t nxt_router_process_port_handlers = { .rpc_ready = nxt_port_rpc_handler, .rpc_error = nxt_port_rpc_handler, .oosm = nxt_router_oosm_handler, + .detached = nxt_router_detached_handler, }; @@ -1725,6 +1734,7 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) app_stat->processes = app->processes; app_stat->unaccounted_processes = app->unaccounted_processes; app_stat->idle_processes = app->idle_processes; + app_stat->detached_processes = app->detached_processes; report->apps_count++; app_stat++; @@ -5603,8 +5613,7 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, nxt_app_t *app; nxt_buf_t *b, *next; nxt_bool_t start_process, unlinked; - nxt_port_t *app_port, *main_app_port, *idle_port; - nxt_queue_link_t *idle_lnk; + nxt_port_t *app_port, *main_app_port; nxt_http_request_t *r; nxt_debug(task, "stream #%uD: got ack from %PI:%d", @@ -5617,7 +5626,6 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, app = req_rpc_data->app; r = req_rpc_data->request; - start_process = 0; unlinked = 0; nxt_thread_mutex_lock(&app->mutex); @@ -5645,41 +5653,7 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, main_app_port = app_port->main_app_port; - if (nxt_queue_chk_remove(&main_app_port->idle_link)) { - app->idle_processes--; - - nxt_debug(task, "app '%V' move port %PI:%d out of %s (ack)", - &app->name, main_app_port->pid, main_app_port->id, - (main_app_port->idle_start ? "idle_ports" : "spare_ports")); - - /* Check port was in 'spare_ports' using idle_start field. */ - if (main_app_port->idle_start == 0 - && app->idle_processes >= app->spare_processes) - { - /* - * If there is a vacant space in spare ports, - * move the last idle to spare_ports. - */ - nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); - - idle_lnk = nxt_queue_last(&app->idle_ports); - idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); - nxt_queue_remove(idle_lnk); - - nxt_queue_insert_tail(&app->spare_ports, idle_lnk); - - idle_port->idle_start = 0; - - nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " - "to spare_ports", - &app->name, idle_port->pid, idle_port->id); - } - - if (nxt_router_app_can_start(app) && nxt_router_app_need_start(app)) { - app->pending_processes++; - start_process = 1; - } - } + start_process = nxt_router_app_port_busy(task, app, main_app_port, "ack"); main_app_port->active_requests++; @@ -6501,6 +6475,127 @@ nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app) } +/* + * Take a port out of the idle economy. Called with app->mutex held; answers + * whether the caller must start a replacement process once it has dropped the + * lock. Does nothing to a port that is not in one of the idle queues. + * + * Two events reach this transition. An acknowledgement, where a worker has + * taken a request the router was still holding for it, and the start of + * detached work, where a worker the router has already parked as idle says it + * is in fact still running. The second is why this is a function rather than + * a block: setting ->detached without unwinding leaves a port that + * nxt_router_app_port_idle() will refuse to insert again and that + * nxt_router_adjust_idle_timer() will still reap, so the flag and the unwind + * have to happen in one critical section. + * + * "reason" is for the debug log only. + */ + +static nxt_bool_t +nxt_router_app_port_busy(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, + const char *reason) +{ + nxt_bool_t start_process; + nxt_port_t *idle_port; + nxt_queue_link_t *idle_lnk; + + start_process = 0; + + if (!nxt_queue_chk_remove(&port->idle_link)) { + return 0; + } + + app->idle_processes--; + + nxt_debug(task, "app '%V' move port %PI:%d out of %s (%s)", + &app->name, port->pid, port->id, + (port->idle_start ? "idle_ports" : "spare_ports"), reason); + + /* Check port was in 'spare_ports' using idle_start field. */ + if (port->idle_start == 0 && app->idle_processes >= app->spare_processes) { + /* + * If there is a vacant space in spare ports, + * move the last idle to spare_ports. + */ + nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); + + idle_lnk = nxt_queue_last(&app->idle_ports); + idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); + nxt_queue_remove(idle_lnk); + + nxt_queue_insert_tail(&app->spare_ports, idle_lnk); + + idle_port->idle_start = 0; + + nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " + "to spare_ports", + &app->name, idle_port->pid, idle_port->id); + } + + if (nxt_router_app_can_start(app) && nxt_router_app_need_start(app)) { + app->pending_processes++; + start_process = 1; + } + + return start_process; +} + + +/* + * Hand a port back to the idle economy, if it is really idle. Called with + * app->mutex held; answers whether the caller must post app->adjust_idle_work + * once it has dropped the lock, which is how the reaper's timer gets armed. + * + * Extracted from nxt_router_app_port_release() because the end of detached + * work reaches the same transition from nxt_router_detached_handler(), and a + * second copy of a block that touches three queues and two counters is how + * they drift apart. + */ + +static nxt_bool_t +nxt_router_app_port_idle(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port) +{ + nxt_bool_t adjust_idle_timer; + + adjust_idle_timer = 0; + + if (port->pair[1] != -1 + && port->active_requests == 0 + && port->active_websockets == 0 + && port->detached == 0 + && port->idle_link.next == NULL) + { + if (app->idle_processes == app->spare_processes + && app->adjust_idle_work.data == NULL) + { + adjust_idle_timer = 1; + app->adjust_idle_work.data = app; + app->adjust_idle_work.next = NULL; + } + + if (app->idle_processes < app->spare_processes) { + nxt_queue_insert_tail(&app->spare_ports, &port->idle_link); + + nxt_debug(task, "app '%V' move port %PI:%d to spare_ports", + &app->name, port->pid, port->id); + + } else { + nxt_queue_insert_tail(&app->idle_ports, &port->idle_link); + + port->idle_start = task->thread->engine->timers.now; + + nxt_debug(task, "app '%V' move port %PI:%d to idle_ports", + &app->name, port->pid, port->id); + } + + app->idle_processes++; + } + + return adjust_idle_timer; +} + + static void nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_apr_action_t action) @@ -6562,37 +6657,7 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_port_inc_use(main_app_port); } - adjust_idle_timer = 0; - - if (main_app_port->pair[1] != -1 - && main_app_port->active_requests == 0 - && main_app_port->active_websockets == 0 - && main_app_port->idle_link.next == NULL) - { - if (app->idle_processes == app->spare_processes - && app->adjust_idle_work.data == NULL) - { - adjust_idle_timer = 1; - app->adjust_idle_work.data = app; - app->adjust_idle_work.next = NULL; - } - - if (app->idle_processes < app->spare_processes) { - nxt_queue_insert_tail(&app->spare_ports, &main_app_port->idle_link); - - nxt_debug(task, "app '%V' move port %PI:%d to spare_ports", - &app->name, main_app_port->pid, main_app_port->id); - } else { - nxt_queue_insert_tail(&app->idle_ports, &main_app_port->idle_link); - - main_app_port->idle_start = task->thread->engine->timers.now; - - nxt_debug(task, "app '%V' move port %PI:%d to idle_ports", - &app->name, main_app_port->pid, main_app_port->id); - } - - app->idle_processes++; - } + adjust_idle_timer = nxt_router_app_port_idle(task, app, main_app_port); nxt_thread_mutex_unlock(&app->mutex); @@ -6622,7 +6687,7 @@ void nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) { nxt_app_t *app; - nxt_bool_t unchain, start_process; + nxt_bool_t unchain, start_process, detached; nxt_port_t *idle_port; nxt_queue_link_t *idle_lnk; @@ -6630,6 +6695,8 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) nxt_assert(app != NULL); + detached = 0; + nxt_thread_mutex_lock(&app->mutex); if (port == app->proto_port) { @@ -6688,6 +6755,21 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) app->processes--; + /* + * A worker can die in the middle of its detached work -- a fatal error + * after the response went out, or a kill. This is the only path that + * runs for it then, so the count has to be settled here or the + * application never reaches the zero nxt_router_free_app() asserts on. + * The matching application reference is dropped below, outside the lock. + */ + + detached = port->detached; + + if (detached) { + port->detached = 0; + app->detached_processes--; + } + start_process = !task->thread->engine->shutdown && nxt_router_app_can_start(app) && nxt_router_app_need_start(app); @@ -6704,6 +6786,12 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) nxt_port_use(task, port, -1); } + /* The reference nxt_router_detached_apply() took for this worker. */ + + if (detached) { + nxt_router_app_use(task, app, -1); + } + if (start_process) { nxt_router_start_app_process(task, app); } @@ -6750,6 +6838,18 @@ nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, void *data) lnk = nxt_queue_first(&app->idle_ports); port = nxt_queue_link_data(lnk, nxt_port_t, idle_link); + /* + * A worker running detached work must never be reachable from here: + * this loop clears ->app, which stops nxt_port_close() from ever + * running nxt_router_app_port_close() for the port, and the count + * and the application reference would then never be settled. + * nxt_router_app_port_idle() refuses to insert such a port and + * nxt_router_app_port_busy() takes one that is already here back + * out, both under this same mutex. + */ + + nxt_assert(port->detached == 0); + timeout = port->idle_start + app->idle_timeout; nxt_debug(task, "app '%V' pid %PI, start %M, timeout %M, threshold %M", @@ -6889,6 +6989,7 @@ nxt_router_free_app(nxt_task_t *task, void *obj, void *data) nxt_assert(app->proto_port == NULL); nxt_assert(app->processes == 0); + nxt_assert(app->detached_processes == 0); nxt_assert(app->active_requests == 0); nxt_assert(app->port_hash_count == 0); nxt_assert(app->idle_processes == 0); @@ -7517,6 +7618,231 @@ nxt_router_http_request_release(nxt_task_t *task, void *obj, void *data) } +/* + * An application says it answered a request and kept running, or that such + * work has finished. PHP's fastcgi_finish_request() is the case this + * exists for: libunit reports the request done while the script runs on, so + * without this the router counts the worker idle, hands its slot back to + * "processes": {"max"}, and reaps a process that is still executing. + * + * Runs on the router's main thread, because the message arrives on the + * router's own port; see nxt_router_detached_handler(). The port is + * identified by pid alone: the accounting hangs off the application's main + * port, the one with id 0, and a worker has exactly one however many + * contexts it runs. Looking it up rather than following ->main_app_port + * from the sender's port is also what keeps this off a stale pointer -- that + * field is never cleared, and the main port can be released while a sibling + * port of the same process is still registered. + * + * Either edge may arrive with the port already in the state being asked + * for, and both then do nothing: a start edge is idempotent because several + * contexts of one worker can be detached at once, and a finish edge because + * the worker's death has already settled the state. + */ + +static void +nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) +{ + int drop; + nxt_app_t *app; + nxt_port_t *port; + nxt_bool_t changed, start_process, adjust_idle_timer; + nxt_runtime_t *rt; + nxt_atomic_uint_t c; + + rt = task->thread->runtime; + + nxt_assert(task->thread->engine == rt->main_engine); + + port = nxt_runtime_port_find(rt, pid, 0); + + if (nxt_slow_path(port == NULL)) { + nxt_debug(task, "detached_handler: %PI has no main port", pid); + return; + } + + app = port->app; + + if (nxt_slow_path(app == NULL)) { + nxt_debug(task, "detached_handler: port %PI:%d has no application", + pid, port->id); + return; + } + + /* + * A reference, taken only if the application still has one. The + * count reaching zero is what posts nxt_router_free_app() to this + * thread, and the last drop can happen on a worker engine while this + * edge is already on its way here: the request that held the + * application is released there, and a reconfiguration has already + * taken the configuration's reference. A plain increment would + * resurrect an application whose free is queued behind this handler, + * and the start it may post below would then lock a mutex in freed + * memory. Zero is treated exactly like no application at all; the + * free that is coming clears ->app and QUITs the worker. + */ + + for ( ;; ) { + c = app->use_count; + + if (c == 0) { + nxt_debug(task, "detached_handler: app '%V' is being freed", + &app->name); + return; + } + + if (nxt_atomic_cmp_set(&app->use_count, c, c + 1)) { + break; + } + } + + nxt_debug(task, "app '%V' port %PI:%d detached state %d", + &app->name, port->pid, port->id, (int) state); + + changed = 0; + start_process = 0; + adjust_idle_timer = 0; + + nxt_thread_mutex_lock(&app->mutex); + + if (state == NXT_PORT_DETACHED_START) { + + if (port->detached == 0) { + port->detached = 1; + app->detached_processes++; + changed = 1; + + /* + * The worker may already have been parked as idle. libunit + * sends the start edge before the last response message, but + * that only orders the two on the wire: the response is + * answered by the engine that owns the request, this by the + * main thread, and a request that was failed rather than + * answered -- a "limits": {"timeout"} expiry, an error -- runs + * nxt_router_app_port_release() with no start edge in sight at + * all, so the port can have been sitting in idle_ports for as + * long as the application chose to keep running. + * + * Unwind it here, exactly as an acknowledgement does. Leaving + * it in place would be worse than not having this message: the + * reaper would QUIT a worker that is still executing -- the bug + * this exists to fix -- and would clear ->app on the way, so + * nxt_router_app_port_close() would never run and the count and + * the application reference below would never be settled. + */ + + start_process = nxt_router_app_port_busy(task, app, port, + "detached"); + } + + } else if (port->detached != 0) { + port->detached = 0; + app->detached_processes--; + changed = 1; + + /* + * Only now may this worker rejoin the idle economy. Its last + * response went out long ago, so nothing else will run this + * transition for it. + */ + + adjust_idle_timer = nxt_router_app_port_idle(task, app, port); + } + + nxt_thread_mutex_unlock(&app->mutex); + + if (adjust_idle_timer) { + nxt_router_app_use(task, app, 1); + nxt_event_engine_post(app->engine, &app->adjust_idle_work); + } + + /* + * Holding the reference taken above: this is the only caller that can + * reach nxt_router_start_app_process() with no reference of its own. + */ + + if (start_process) { + nxt_router_start_app_process(task, app); + } + + /* + * A start edge that changed the state keeps the reference taken above: + * it is what keeps this worker's application alive for as long as the + * work runs. A detached worker holds no request, and a request is what + * otherwise holds the application -- by the time the work starts + * nxt_request_rpc_data_unlink() has already dropped its reference, so a + * configuration reload could free the application out from under a + * process still executing. The finish edge that changed the state + * returns it, and an edge that changed nothing returns only its own, so + * a repeated start cannot take a second reference that the single + * finish never returns. + */ + + drop = 1; + + if (changed) { + drop = (state == NXT_PORT_DETACHED_START) ? 0 : 2; + } + + if (drop != 0) { + nxt_router_app_use(task, app, -drop); + } +} + + +/* + * The edge arrives on the router's own port, whichever engine answered the + * request: libunit sends both edges to its router port, the way it sends + * OOSM. That port is read on the main thread, which is where everything + * the edge touches lives -- rt->ports is read without a lock, which is only + * safe on the thread that adds to and removes from it, and an application + * is freed on app->engine, the same thread. One socket per worker also + * keeps a finish edge behind the start edge that preceded it, which the + * ports of two engines could not: their reads race, and the flag would be + * cleared under the next request's work. + */ + +static void +nxt_router_detached_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + /* + * The pid in the message is chosen by the sender, and every worker holds + * a copy of this port. Accept an edge only for the process that sent it, + * or a worker could pin another worker as detached or release one. The + * router port has no shared memory queue, so the message always comes + * from the socket and carries the kernel's credentials. + */ + if (nxt_slow_path(nxt_recv_msg_cmsg_pid(msg) != msg->port_msg.pid)) { + nxt_alert(task, "process %PI sent a detached edge for process %PI", + nxt_recv_msg_cmsg_pid(msg), msg->port_msg.pid); + return; + } + + if (nxt_slow_path(msg->buf == NULL + || nxt_buf_used_size(msg->buf) < (int) sizeof(uint8_t))) + { + nxt_alert(task, "detached_handler: %PI sent no state byte", + msg->port_msg.pid); + return; + } + + nxt_router_detached_apply(task, msg->port_msg.pid, *msg->buf->mem.pos); +} + + +#if (NXT_TESTS) + +/* For src/test/nxt_router_detached_test.c. */ + +void +nxt_router_test_detached_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_router_detached_handler(task, msg); +} + +#endif + + static void nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { diff --git a/src/nxt_router.h b/src/nxt_router.h index 2bff5332d..8c3bdf3a4 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -161,6 +161,15 @@ struct nxt_app_s { uint32_t processes; uint32_t idle_processes; + /* + * Workers that answered a request and kept running. Counted in + * ->processes like any live worker -- a subset of it, not a separate + * population -- but never in ->idle_processes, because the reaper walks + * idle_ports and asserts that queue is non-empty while idle_processes + * exceeds spare_processes. + */ + uint32_t detached_processes; + /* * Application processes the router asked for, that were forked, and that * it has neither a port nor a pid for: the ones a "limits": @@ -344,6 +353,12 @@ void nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, void nxt_router_access_log_reopen_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +#if (NXT_TESTS) +/* The detached edge handler, for the sender check test. */ +void nxt_router_test_detached_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +#endif + #if (NXT_TESTS) /* * The config-apply start and the temporary configuration its failure path diff --git a/src/nxt_status.c b/src/nxt_status.c index 0eb0c696a..213c05792 100644 --- a/src/nxt_status.c +++ b/src/nxt_status.c @@ -43,6 +43,7 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) static const nxt_str_t run_str = nxt_string("running"); static const nxt_str_t start_str = nxt_string("starting"); static const nxt_str_t unacc_str = nxt_string("unaccounted"); + static const nxt_str_t detached_str = nxt_string("detached"); /* * modules, connections, requests, applications -- plus "telemetry" when @@ -212,7 +213,7 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) return NULL; } - obj = nxt_conf_create_object(mp, 4); + obj = nxt_conf_create_object(mp, 5); if (nxt_slow_path(obj == NULL)) { return NULL; } @@ -224,6 +225,8 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) nxt_conf_set_member_integer(obj, &unacc_str, app->unaccounted_processes, 2); nxt_conf_set_member_integer(obj, &idle_str, app->idle_processes, 3); + nxt_conf_set_member_integer(obj, &detached_str, + app->detached_processes, 4); obj = nxt_conf_create_object(mp, 1); if (nxt_slow_path(obj == NULL)) { diff --git a/src/nxt_status.h b/src/nxt_status.h index f8eb1c8ed..cdfb6fb3a 100644 --- a/src/nxt_status.h +++ b/src/nxt_status.h @@ -14,6 +14,7 @@ typedef struct { uint32_t processes; uint32_t unaccounted_processes; uint32_t idle_processes; + uint32_t detached_processes; } nxt_status_app_t; diff --git a/src/nxt_unit.c b/src/nxt_unit.c index ded5187f7..e50ef7f1d 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -77,6 +77,8 @@ static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx); static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq); +static void nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx); +static int nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state); static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); static int nxt_unit_request_check_response_port(nxt_unit_request_info_t *req, @@ -429,6 +431,14 @@ struct nxt_unit_ctx_impl_s { uint8_t ready; /* 1 bit */ uint8_t quit_param; + /* + * The application answered a request with + * nxt_unit_request_done_detached() and has not returned from its + * request handler yet. Holds the router's view of this worker as + * busy, and holds off a graceful quit, until it does. + */ + uint8_t detached; /* 1 bit */ + nxt_unit_mmap_buf_t ctx_buf[2]; nxt_unit_read_buf_t ctx_read_buf; @@ -802,6 +812,16 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, ctx_impl->ready = 0; ctx_impl->quit_param = NXT_QUIT_GRACEFUL; + /* + * Explicitly, like every field above it: this function initialises the + * context field by field and never memsets it, so anything left out + * starts as whatever the allocator left behind -- 0xAA in a debug + * build, which reads as "detached" and sends the finish report to a + * poison pointer. + */ + + ctx_impl->detached = 0; + nxt_queue_init(&ctx_impl->free_req); nxt_queue_init(&ctx_impl->free_ws); nxt_queue_init(&ctx_impl->active_req); @@ -1613,6 +1633,8 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, if (preq == NULL) { lib->callbacks.request_handler(req); + nxt_unit_ctx_detached_done(ctx); + } else { *preq = req; } @@ -1669,6 +1691,8 @@ nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) if (req->content_fd != -1 || l == req->content_length) { lib->callbacks.request_handler(req); + + nxt_unit_ctx_detached_done(ctx); } return NXT_UNIT_OK; @@ -3496,6 +3520,124 @@ nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size) } +/* + * Tell the router this worker is, or is no longer, running work of its own + * after a response. One byte of payload says which edge; see + * nxt_port_detached_t in src/nxt_port.h for why it is a byte and a new + * message type rather than a flag on the header. + * + * Sent to the router's own port, the way OOSM is, not to the port the + * response went to. The router reads that port on its main thread, which + * is the thread that owns the accounting, and one socket keeps the finish + * edge behind the start edge that preceded it: the next request's response + * may belong to another router engine, and two engines' reads of two + * sockets race. The router looks the worker up by pid. + */ + +static int +nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state) +{ + int res; + nxt_unit_impl_t *lib; + struct { + nxt_port_msg_t msg; + uint8_t state; + } m; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + + if (nxt_slow_path(lib->router_port == NULL)) { + nxt_unit_debug(ctx, "detached %d: no router port to report on", + (int) state); + return NXT_UNIT_ERROR; + } + + memset(&m, 0, sizeof(m)); + + m.msg.pid = lib->pid; + m.msg.type = _NXT_PORT_MSG_DETACHED; + m.msg.last = 1; + m.state = state; + + res = nxt_unit_port_send(ctx, lib->router_port, &m, sizeof(m), NULL); + if (nxt_slow_path(res != sizeof(m))) { + return NXT_UNIT_ERROR; + } + + return NXT_UNIT_OK; +} + + +void +nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(req->ctx, nxt_unit_ctx_impl_t, ctx); + + /* + * Before the response, not after. The two go to different ports, so + * this orders nothing on the router's side by itself -- the router + * takes a worker it has already parked as idle back out when the edge + * arrives -- but it does keep the window in which the port sits in the + * idle queue as short as the router's own read makes it. + * + * A failure here is not fatal to the request: the response still goes + * out below, and the worst case is the accounting this call was meant + * to correct, which is what the router does today for every such + * response. + */ + + if (ctx_impl->detached == 0) { + if (nxt_unit_send_detached(req->ctx, NXT_PORT_DETACHED_START) + == NXT_UNIT_OK) + { + ctx_impl->detached = 1; + + } else { + nxt_unit_req_alert(req, "failed to report a detached response"); + } + } + + nxt_unit_request_done(req, rc); +} + + +/* + * The application's request handler has returned, so whatever it was doing + * after its response is over. Runs on every return path -- a normal return, + * and the ones PHP reaches through exit() and a fatal error, which all come + * back through the handler call site. + */ + +static void +nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + if (ctx_impl->detached == 0) { + return; + } + + ctx_impl->detached = 0; + + (void) nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_FINISH); + + /* + * A graceful QUIT that arrived during the work was deferred by + * nxt_unit_quit() on this flag, the way one that arrives during a + * request is deferred on active_req. That one is retried when the + * request is released; this is the equivalent. + */ + + if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) { + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); + } +} + + void nxt_unit_request_done(nxt_unit_request_info_t *req, int rc) { @@ -5129,6 +5271,8 @@ nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) lib->callbacks.request_handler(&req_impl->req); + nxt_unit_ctx_detached_done(ctx); + } nxt_queue_loop; } @@ -6100,7 +6244,8 @@ nxt_unit_quit(nxt_unit_ctx_t *ctx, uint8_t quit_param) quit = nxt_queue_is_empty(&ctx_impl->active_req) && nxt_queue_is_empty(&ctx_impl->pending_rbuf) - && ctx_impl->wait_items == 0; + && ctx_impl->wait_items == 0 + && ctx_impl->detached == 0; pthread_mutex_unlock(&ctx_impl->mutex); diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 46d6f3c6b..35271acf8 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -314,6 +314,25 @@ ssize_t nxt_unit_request_readline_size(nxt_unit_request_info_t *req, void nxt_unit_request_done(nxt_unit_request_info_t *req, int rc); +/* + * Finish a request the way nxt_unit_request_done() does, and tell the router + * that this worker is still running: the application has answered the client + * but has not returned, so the process is busy even though it has no request. + * + * Without this the router counts such a worker idle the moment the response + * goes out. It then hands the worker's slot back to "processes": {"max"}, + * and its idle timer can reap a process that is still executing. + * + * libunit reports the work finished by itself, once the request handler + * returns; the application does not have to pair this call with anything. + * That covers the requests libunit hands to the handler, not one taken with + * nxt_unit_dequeue_request(). The router keeps one flag per worker, not a + * count per context, so a worker running several contexts at once has the + * first context's finish clear it. PHP's fastcgi_finish_request() is the + * caller this exists for, and PHP runs one context. + */ +void nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc); + int nxt_unit_websocket_send(nxt_unit_request_info_t *req, uint8_t opcode, uint8_t last, const void *start, size_t size); diff --git a/src/test/nxt_router_detached_test.c b/src/test/nxt_router_detached_test.c new file mode 100644 index 000000000..6e804af0c --- /dev/null +++ b/src/test/nxt_router_detached_test.c @@ -0,0 +1,223 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the sender check in nxt_router_detached_handler() + * (src/nxt_router.c). + * + * Every application worker holds a copy of the router port, and the pid in + * a port message is chosen by the sender. A detached edge that named + * another worker's pid used to be applied to that worker: a forged start + * pinned it busy together with a reference to its application, and a forged + * finish let the reaper take a worker that was still running. The edge is + * now applied only when the kernel's credential names the same process. + * + * The fixture is one application with the main port of one worker. The port + * has no socket (pair[1] is -1) and is in no idle queue, so the edges change + * only ->detached, the count and the application reference, and those are + * what the test reads. Each forged edge is followed by the genuine one, which + * shows that the fixture reaches the accounting at all. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#if (NXT_USE_CMSG_PID) + +static nxt_int_t +nxt_router_detached_test_edge(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, nxt_app_t *app, uint8_t state, nxt_pid_t sender, + nxt_bool_t detached, nxt_atomic_t use_count, const char *name) +{ + nxt_buf_t b; + nxt_port_recv_msg_t msg; + + nxt_memzero(&b, sizeof(nxt_buf_t)); + b.mem.start = &state; + b.mem.pos = &state; + b.mem.free = &state + 1; + b.mem.end = &state + 1; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + msg.buf = &b; + msg.size = 1; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.pid = port->pid; + msg.port_msg.type = _NXT_PORT_MSG_DETACHED; + msg.port_msg.last = 1; + msg.cmsg_pid = sender; + + nxt_router_test_detached_handler(task, &msg); + + if (port->detached != detached + || app->detached_processes != (detached ? 1 : 0) + || app->use_count != use_count) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router detached test: %s: detached %d, count %uD, " + "use count %d; expected %d, %d, %d", + name, (int) port->detached, app->detached_processes, + (int) app->use_count, (int) detached, + detached ? 1 : 0, (int) use_count); + return NXT_ERROR; + } + + return NXT_OK; +} + +#endif + + +nxt_int_t +nxt_router_detached_test(nxt_thread_t *thr) +{ +#if !(NXT_USE_CMSG_PID) + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router detached test skipped: no sender credentials"); + return NXT_OK; + +#else + + nxt_mp_t *mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_pid_t pid, other; + nxt_task_t *task; + nxt_port_t *port; + nxt_bool_t app_mutex, hashed; + nxt_runtime_t *rt, *saved_rt; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router detached test started"); + + ret = NXT_ERROR; + port = NULL; + app_mutex = 0; + hashed = 0; + + task = thr->task; + task->thread = thr; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + app = nxt_mp_zalloc(mp, sizeof(nxt_app_t)); + if (nxt_slow_path(rt == NULL || app == NULL)) { + goto done; + } + + rt->mem_pool = mp; + + /* The handler runs on the main engine only. */ + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "detached-test"); + + /* Never reached by the edges below, so nothing frees the application. */ + app->use_count = 8; + + /* A pid that is not this process, so a sender of nxt_pid is foreign. */ + + pid = nxt_pid + 1; + other = nxt_pid; + + port = nxt_port_new(task, 0, pid, NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + port->app = app; + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, port) != NXT_OK)) { + goto done; + } + + hashed = 1; + + if (nxt_router_detached_test_edge(thr, task, port, app, + NXT_PORT_DETACHED_START, other, 0, 8, + "a forged start") + != NXT_OK + || nxt_router_detached_test_edge(thr, task, port, app, + NXT_PORT_DETACHED_START, pid, 1, 9, + "a genuine start") + != NXT_OK + || nxt_router_detached_test_edge(thr, task, port, app, + NXT_PORT_DETACHED_FINISH, other, 1, 9, + "a forged finish") + != NXT_OK + || nxt_router_detached_test_edge(thr, task, port, app, + NXT_PORT_DETACHED_FINISH, -1, 1, 9, + "a finish with no credential") + != NXT_OK + || nxt_router_detached_test_edge(thr, task, port, app, + NXT_PORT_DETACHED_FINISH, pid, 0, 8, + "a genuine finish") + != NXT_OK) + { + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router detached test passed"); + + ret = NXT_OK; + +done: + + if (hashed) { + (void) nxt_port_hash_remove(&rt->ports, port); + } + + if (port != NULL) { + port->app = NULL; + nxt_port_use(task, port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_mp_destroy(mp); + + return ret; + +#endif +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 8208896d6..1da41a362 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -234,6 +234,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_detached_test(thr) != NXT_OK) { + return 1; + } + if (nxt_main_start_process_reply_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 94fa6e331..5de282a73 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -80,6 +80,7 @@ nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_death_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); +nxt_int_t nxt_router_detached_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); From d717041a2c53ef1adaa6c89e3161a9ca7a4fd45c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 07:23:33 +0200 Subject: [PATCH 216/391] php: report fastcgi_finish_request() as a detached response fastcgi_finish_request() answers the client and returns to the script, which is exactly the case the previous commit added a signal for. It now uses nxt_unit_request_done_detached(), so the router keeps counting this worker while the script runs instead of parking it as idle and reaping it. Nothing else in the module changes. libunit sends the finish edge when the request handler returns, and PHP reaches that return through exit() and through a fatal error as well as through an ordinary one, so the exit paths need no audit of their own -- and the test says so rather than the reasoning. The tests measure the process, not the accounting that is under test: worker_pids() counts the prototype's children, the way test_app_lifecycle.py does, so "one live PHP child" means one real process. test_php_detached_work_does_not_exceed_max is red without the previous commits -- measured at two live children under "max": 1 -- and red in a different way on origin/master, where the parked request is never answered at all. Eight cases: that the response really does overlap the script, so the rest cannot pass against a build where fastcgi_finish_request() did nothing; the strict bound; the transition back once the work ends; the deadline, asserting the timed-out request never runs, before or after the first script returns; a start edge landing on a worker the router has already parked as idle, which a "limits": {"timeout"} expiry produces on demand; repetition across several idle-timeout intervals, which is the shape that turns a brief overshoot into an unbounded one; and two lifecycle edges -- exit() and a fatal error after the response, and removing the configuration while a worker is still running. The suite drains before each test ends, polling "/status" for the detached count rather than sleeping: a worker that is still detached at teardown is deliberately not reaped, so one left running would be counted by the next test and its port descriptors flagged as a leak. That polling is also what caught the first version of this change, which emitted the finish edge from one of libunit's three request-handler call sites and pinned the worker out of the idle economy for good. Co-Authored-By: Claude Opus 5 --- src/nxt_php_sapi.c | 11 +- test/php/detached_worker/index.php | 64 +++++ test/test_php_detached_max.py | 443 +++++++++++++++++++++++++++++ 3 files changed, 517 insertions(+), 1 deletion(-) create mode 100644 test/php/detached_worker/index.php create mode 100644 test/test_php_detached_max.py diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index e56e9cd8e..30c43b7a7 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -267,7 +267,16 @@ PHP_FUNCTION(fastcgi_finish_request) #endif } - nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + /* + * Detached, not plain done: the script keeps running after this returns. + * Without telling the router that, the worker is counted idle from here + * -- its slot goes back to "processes": {"max"} and its idle timer can + * reap a process that is still executing. libunit reports the work + * finished by itself when this request handler returns, which covers the + * exit() and fatal paths as well as an ordinary return. + */ + + nxt_unit_request_done_detached(ctx->req, NXT_UNIT_OK); ctx->req = NULL; PG(connection_status) = PHP_CONNECTION_ABORTED; diff --git a/test/php/detached_worker/index.php b/test/php/detached_worker/index.php new file mode 100644 index 000000000..434fc672c --- /dev/null +++ b/test/php/detached_worker/index.php @@ -0,0 +1,64 @@ + 0) { + sleep($before); +} + +echo "done"; + +fastcgi_finish_request(); + +$sleep = isset($_GET['sleep']) ? (int) $_GET['sleep'] : 0; + +if ($sleep > 0) { + sleep($sleep); +} + +$done = isset($_GET['done']) ? (string) $_GET['done'] : ''; + +if ($done !== '') { + file_put_contents($done, (string) getmypid()); +} + +$after = isset($_GET['after']) ? (string) $_GET['after'] : ''; + +if ($after === 'exit') { + exit(0); +} + +if ($after === 'fatal') { + /* An uncaught Error: a fatal path that runs after the response went out. */ + throw new Error('detached fatal'); +} diff --git a/test/test_php_detached_max.py b/test/test_php_detached_max.py new file mode 100644 index 000000000..7f8f7f121 --- /dev/null +++ b/test/test_php_detached_max.py @@ -0,0 +1,443 @@ +"""processes.max must bound live workers, detached post-response work included. + +fastcgi_finish_request() reports the request done while the script keeps +running. The router takes that as the response, so the worker is counted +idle and becomes reapable while it is in fact still executing PHP. + +These tests assert the property that follows from "max" being a cap on +processes rather than on requests: however the router accounts for a worker +that is finishing detached work, the number of live PHP children of the +prototype must never exceed "max". +""" + +import os +import subprocess +import threading +import time + +from unit.applications.lang.php import ApplicationPHP +from unit.control import Control +from unit.option import option + +prerequisites = {'modules': {'php': 'any'}} + +client = ApplicationPHP() + +IDLE_TIMEOUT = 2 + +# Comfortably past IDLE_TIMEOUT, so "the reaper did not act" is unambiguous, +# and short enough that every test can wait for the work to finish before it +# ends. Draining matters now: a detached worker is deliberately not reaped, +# so one left running would be counted by the next test and its port +# descriptors flagged as a leak by conftest. +SLEEP_SECONDS = 6 + + +def worker_pids(name='detached_worker'): + """Live worker pids, as the children of the application's prototype. + + Same derivation as test_app_lifecycle.py: the prototype's title survives + for every runtime, and its children are the workers. A worker that has + been sent QUIT but has not exited is still a child, which is the whole + point here -- it is still a live PHP process holding memory and a slot. + """ + + output = subprocess.check_output( + ['ps', 'ax', '-o', 'pid=', '-o', 'ppid=', '-o', 'args='] + ).decode() + + rows = [p for p in (l.split(None, 2) for l in output.splitlines()) + if len(p) == 3] + + marker = f'unit: "{name}" prototype' + prototypes = {pid for pid, _, args in rows if marker in args} + + if not prototypes: + return set() + + return {pid for pid, ppid, _ in rows if ppid in prototypes} + + +def app_processes(name='detached_worker'): + """The application's "processes" object from /status.""" + + status = Control().conf_get('/status') + apps = status.get('applications', {}) + + return apps.get(name, {}).get('processes', {}) + + +def drain(timeout=30): + """Wait for detached work to finish and the workers to settle. + + Polls the router's own view rather than sleeping a fixed time: the + "detached" count is the thing under test, so waiting on it also asserts + it reaches zero on every ordinary path. + """ + + deadline = time.time() + timeout + + while time.time() < deadline: + if app_processes().get('detached', 0) == 0: + return True + + time.sleep(0.2) + + return False + + +def marker(name): + """An absolute path under the test temp dir, removed if it exists.""" + + path = f'{option.temp_dir}/{name}' + + if os.path.exists(path): + os.remove(path) + + return path + + +def test_php_detached_response_overlaps_the_script(): + """The response really does arrive while the script is still running. + + Without this the other tests could pass against a build where + fastcgi_finish_request() did nothing, because every request would simply + be serialised. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + start = time.time() + assert client.get(url=f'/?sleep={SLEEP_SECONDS}')['status'] == 200 + elapsed = time.time() - start + + assert elapsed < SLEEP_SECONDS / 2, ( + f'the response took {elapsed:.1f}s, so it waited for the script ' + f'rather than overlapping it' + ) + + assert len(worker_pids()) == 1, 'one worker serves the detached request' + + assert drain(), 'the detached work finished and the count cleared' + + +def test_php_detached_work_does_not_exceed_max(): + """A parked request must not add a second live PHP child under max: 1. + + The first script detaches and sleeps well past idle_timeout. A second + request arrives while it runs, so it parks. Whatever the router does + about that parked request, "max": 1 forbids a second live worker while + the first is still executing. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + assert client.get(url=f'/?sleep={SLEEP_SECONDS}')['status'] == 200 + + first = worker_pids() + assert len(first) == 1, 'one worker after the detached request' + + # Park a second request against the still-busy worker. It is issued from + # a thread because it will not be answered until capacity exists. + threading.Thread( + target=lambda: client.get(read_timeout=SLEEP_SECONDS + 5), + daemon=True, + ).start() + + # Past idle_start + idle_timeout, where the reaper acts on the worker it + # believes is idle. + deadline = time.time() + IDLE_TIMEOUT * 3 + 2 + peak = set() + + while time.time() < deadline: + peak |= worker_pids() + + assert len(worker_pids()) <= 1, ( + f'{len(worker_pids())} live PHP children under "max": 1 -- ' + f'a worker running detached work was replaced rather than ' + f'counted' + ) + + time.sleep(0.2) + + assert first <= peak, 'the original worker stayed alive throughout' + + assert drain(), 'the detached work finished and the count cleared' + + +def test_php_detached_completion_releases_the_worker(): + """When the detached work ends, the worker rejoins the idle economy. + + The parked request must then be served -- and served without the router + having started a second process while the first was still running. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + done = marker('detached_done') + short = 6 + + assert client.get(url=f'/?sleep={short}&done={done}')['status'] == 200 + + served = {} + + def parked(): + served['resp'] = client.get(read_timeout=short + 15) + + thread = threading.Thread(target=parked, daemon=True) + thread.start() + + # While the detached work runs the worker is neither idle nor reapable. + time.sleep(IDLE_TIMEOUT * 2) + + procs = app_processes() + assert procs.get('detached', 0) == 1, ( + f'the worker is not reported detached: {procs}' + ) + assert procs.get('idle', 0) == 0, f'a busy worker is reported idle: {procs}' + assert len(worker_pids()) == 1, 'still one live worker' + + thread.join(timeout=short + 20) + + assert served.get('resp', {}).get('status') == 200, ( + 'the parked request was served once capacity existed' + ) + assert os.path.exists(done), 'the detached work ran to completion' + + # The detached state clears once the script returns. + deadline = time.time() + 10 + while time.time() < deadline and app_processes().get('detached', 0) != 0: + time.sleep(0.2) + + assert app_processes().get('detached', 0) == 0, 'detached count cleared' + + +def test_php_detached_parked_request_times_out(): + """limits.timeout bounds the wait for capacity, and the request never runs. + + The second request cannot be served until the detached work ends, so a + timeout below the detached duration must answer 503 -- and the script for + that request must never execute, before or after the first one returns. + """ + + ran = marker('timed_out_ran') + timeout = 3 + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + limits={'timeout': timeout}, + ) + + assert client.get(url=f'/?sleep={SLEEP_SECONDS}')['status'] == 200 + + start = time.time() + resp = client.get(url=f'/?ran={ran}', read_timeout=timeout + 10) + elapsed = time.time() - start + + assert resp['status'] == 503, f'parked request timed out, got {resp}' + assert elapsed < timeout + 5, f'answered at the deadline, took {elapsed:.1f}s' + assert not os.path.exists(ran), 'the timed-out request never executed' + + # And it must still not run once the detached worker becomes free. + time.sleep(3) + assert not os.path.exists(ran), ( + 'the timed-out request executed after the worker freed up' + ) + + assert drain(), 'the detached work finished and the count cleared' + + +def test_php_detached_start_after_the_worker_went_idle(): + """A detached start for a worker the router has already parked as idle. + + The start edge is sent before the last response message, but that only + orders the two on the wire. A request that is failed rather than + answered releases its worker with no start edge in sight at all: + "limits": {"timeout"} answers 503 while the script is still running, and + the release puts the port in idle_ports. The start edge then arrives for + a port that is already there. + + Unless it is taken back out, the reaper QUITs a process that is still + executing PHP -- the exact failure this feature exists to prevent -- and + clears the port's application on the way, after which nothing settles the + detached state: nxt_port_close() skips nxt_router_app_port_close() for a + port with no application, the finish edge is ignored for the same reason, + and the count sticks for the life of the application. + """ + + timeout = 2 + idle_timeout = 6 + + # Answer after the deadline has failed the request, and well before the + # reaper's own deadline, so the start edge really does land on a port + # sitting in idle_ports. + before = timeout + 2 + + # And keep running past the reap, so a reaped worker is one that was + # killed mid-script. + after = idle_timeout + 4 + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': idle_timeout}, + limits={'timeout': timeout}, + ) + + done = marker('detached_late_start') + + start = time.time() + resp = client.get( + url=f'/?before={before}&sleep={after}&done={done}', + read_timeout=before + 10, + ) + + assert resp['status'] == 503, f'the request outlived limits.timeout: {resp}' + + first = worker_pids() + assert len(first) == 1, 'one worker is running the script' + + # After the start edge, before the reaper's deadline. + while time.time() - start < before + 2: + time.sleep(0.2) + + procs = app_processes() + + assert procs.get('detached', 0) == 1, ( + f'the worker did not report itself detached: {procs}' + ) + assert procs.get('idle', 0) == 0, ( + f'a worker running detached work is still counted idle and is ' + f'reapable: {procs}' + ) + + # Past the reaper's deadline, with the script still running. + while time.time() - start < before + idle_timeout + 2: + assert len(worker_pids()) <= 1, '"max": 1 still bounds live workers' + time.sleep(0.2) + + assert worker_pids() == first, 'the worker was not replaced' + + procs = app_processes() + assert procs.get('running', 0) == 1, ( + f'the worker was reaped while it was still running: {procs}' + ) + + assert drain(timeout=after + 20), ( + 'the detached state cleared once the script returned' + ) + assert os.path.exists(done), 'the detached work ran to completion' + + +def test_php_detached_repeated_requests_hold_the_bound(): + """Repeated detached requests must not accumulate live workers. + + This is the shape that turns a brief overshoot into an unbounded one: if + each reap starts a replacement while the previous worker is still running + PHP, one extra child appears per idle_timeout for as long as the scripts + run. Spans several intervals for that reason. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + peak = 0 + rounds = 6 + + for _ in range(rounds): + client.get(url=f'/?sleep={SLEEP_SECONDS}', read_timeout=10) + + for _ in range(int(IDLE_TIMEOUT * 5)): + live = len(worker_pids()) + peak = max(peak, live) + + assert live <= 1, ( + f'{live} live PHP children under "max": 1 after repeated ' + f'detached requests' + ) + + time.sleep(0.2) + + assert peak == 1, f'the application kept exactly one worker, saw {peak}' + + assert drain(), 'the detached work finished and the count cleared' + + +def test_php_detached_exit_and_fatal_still_report_finished(): + """exit() and a fatal error after the response still end the detached state. + + libunit reports the work finished when the request handler returns, and + PHP reaches that return through its bailout as well as through an + ordinary return. If it did not, the worker would stay pinned out of the + idle economy for good -- a worse failure than the one this feature + exists to fix, because nothing would ever clear it. + """ + + for after in ('exit', 'fatal'): + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + done = marker(f'detached_{after}') + + assert ( + client.get(url=f'/?sleep=2&done={done}&after={after}')['status'] + == 200 + ), f'{after}: the response went out' + + assert drain(), f'{after}: the detached state cleared' + assert os.path.exists(done), f'{after}: the detached work ran' + + # And the application still works afterwards. + assert client.get(url='/')['status'] == 200, f'{after}: still serving' + + assert drain(), f'{after}: cleared again' + + +def test_php_detached_survives_configuration_removal(): + """Removing the application while a worker runs detached work is safe. + + A detached worker holds no request, and a request is what otherwise + keeps the application alive, so the detached state has to hold its own + reference. Without it the application is freed under a running process. + The assertion the router itself makes is in nxt_router_free_app(), which + is a debug build; here the observable is that the control plane survives + and the work still completes. + """ + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + ) + + done = marker('detached_removed') + + assert client.get(url=f'/?sleep=4&done={done}')['status'] == 200 + + # Remove the whole configuration while the script is still running. The + # application cannot be deleted on its own while a listener still routes + # to it, so the listener goes with it. + assert 'success' in client.conf( + {'listeners': {}, 'applications': {}}, '/config' + ) + + # The control plane is still answering, and the router did not die with it. + assert 'applications' in Control().conf_get('/status') + + deadline = time.time() + 20 + while time.time() < deadline and not os.path.exists(done): + time.sleep(0.2) + + assert os.path.exists(done), 'the detached work finished after removal' From a5849447f68beab4b41b4ce4a27fae465b6e816a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 14 Sep 2026 07:26:57 +0200 Subject: [PATCH 217/391] router: bound a request that is parked with no worker to ack it A request that finds every process busy is parked in app->ack_waiting_req with its headers already in the shared queue. Both dispatch paths arm r->timer only once a worker acknowledges the request, so a parked one had no deadline of its own: it waited for a worker that might never ask for it. With the previous commits that wait became worth bounding. A worker running detached work is no longer reaped out from under its own script, so a request parked behind it waits exactly as long as the application chooses to run. "limits": {"timeout"} now covers that too -- it bounds waiting for capacity, not only the time a worker spends on a request. The default stays 0, so nothing is bounded that was not bounded before. Arming the timer opens a race the acknowledged deadline does not have, so the handler now tells the two apart. nxt_app_queue_cancel() is a CAS (src/nxt_app_queue.h) that answers false exactly when a worker claimed the slot first, and nothing used to consult it: nxt_router_msg_cancel() took the answer only for ->is_port_mmap_sent, its caller discarded it, and nxt_router_app_timeout() answered 503 before the unlink regardless. A request that lost that race was failed and executed -- and executed twice if the client retried. nxt_router_msg_retract() now does the CAS and nothing else, and the answer is memoised on nxt_msg_info_t. Three things made that shape necessary rather than merely tidy: - The retraction cannot be folded into nxt_router_msg_cancel(), because the chain's tail is the request body that nxt_router_req_headers_ack_handler() still has to send. Completing it on a lost retraction would silently drop the body of every POST that hit the window. - The CAS cannot be asked twice. A win and a loss both leave ->tracking at 0, so a second ask always answers false and would set ->is_port_mmap_sent on a chunk no worker will hand back. - A claimed request must not be answered until its acknowledgement, at this expiry or a later one, and the timer is not re-armed: the acknowledgement arms it again. libunit acknowledges the headers and keeps a request whose body did not fit in the headers message until REQ_BODY arrives. An unlink before the acknowledgement completes that body and cancels the RPC, so the late acknowledgement is ignored and the worker keeps the request for ever. The request is not acknowledged while ->app_port is still the shared port. A worker that claims a request and dies before it acknowledges leaves the request waiting, as it did before this commit. src/test/nxt_router_app_timeout_test.c drives the CAS rather than racing it: the test plays the worker, calling nxt_app_queue_recv() and nxt_app_queue_cancel() itself, so the outcome is chosen. Three legs -- claimed is not answered, its message is intact and the timer is not re-armed; claimed with a late acknowledgement is still not answered by later expiries; unclaimed is answered and a worker arriving afterwards is refused. Mutation-verified: the handler answering unconditionally fails with "a claimed request was answered", dropping the memo so the CAS runs twice fails with "an unclaimed request was not answered", and re-arming and answering at the next expiry fails with "the deadline was re-armed" (and, with that check removed, "answered before its acknowledgement"). Found by unit-79 from a Codex finding on #369. An earlier version of this commit asserted the opposite -- that a request timed out while parked never runs -- which was wrong. Taken from #369, which found the deadline; the reaper change that PR pairs it with is what the previous commits replace. Co-Authored-By: Claude Opus 5 --- CHANGES | 13 + auto/sources | 1 + docs/changes.xml | 20 + src/nxt_router.c | 143 ++++++- src/nxt_router.h | 11 + src/nxt_router_request.h | 17 + src/test/nxt_router_app_timeout_test.c | 546 +++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/test_php_detached_max.py | 7 +- 10 files changed, 746 insertions(+), 17 deletions(-) create mode 100644 src/test/nxt_router_app_timeout_test.c diff --git a/CHANGES b/CHANGES index ffc109500..685ce21bb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,19 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: an application that answered a request and kept running was + counted idle. PHP does this whenever a script calls + fastcgi_finish_request(): the response goes out, the script runs on, + and the router returned the worker's slot to "processes": {"max"} and + let its idle timer reap a process that was still executing. Such a + worker now reports itself busy until the work ends, so "max" bounds + live processes and "/status" reports them as "detached". + + *) Change: "limits": {"timeout"} now also bounds how long a request waits + for an application process, not only how long a worker may spend on + it. A request that waited for capacity previously had no deadline at + all. The default remains no timeout. + *) Security: the string form of "access_log" "format" wrote what a variable expanded to without escaping it. "$uri" is the percent-decoded target, so a request for "/%0d%0a..." put a real diff --git a/auto/sources b/auto/sources index f63350349..ea4b893ab 100644 --- a/auto/sources +++ b/auto/sources @@ -177,6 +177,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_proto_wedge_test.c \ src/test/nxt_router_proto_death_test.c \ src/test/nxt_router_start_timeout_test.c \ + src/test/nxt_router_app_timeout_test.c \ src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_router_detached_test.c \ src/test/nxt_main_start_process_reply_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 33ce15483..e1bb596af 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,26 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +an application that answered a request and kept running was counted idle. PHP +does this whenever a script calls fastcgi_finish_request(): the response goes +out, the script runs on, and the router returned the worker's slot to +"processes": {"max"} and let its idle timer reap a process that was still +executing. Such a worker now reports itself busy until the work ends, so "max" +bounds live processes and "/status" reports them as "detached". + + + + + +"limits": {"timeout"} now also bounds how long a request waits for an +application process, not only how long a worker may spend on it. A request +that waited for capacity previously had no deadline at all. The default +remains no timeout. + + + the string form of "access_log" "format" wrote what a variable expanded to diff --git a/src/nxt_router.c b/src/nxt_router.c index 4d1efef26..471ada738 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -23,8 +23,6 @@ #include #include -#define NXT_SHARED_PORT_ID 0xFFFFu - #if (NXT_HAVE_OTEL) #define NXT_OTEL_BATCH_DEFAULT 128 #define NXT_OTEL_SAMPLING_DEFAULT 1 @@ -1168,36 +1166,70 @@ nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app) } +/* + * Take the queued request message back from the shared port queue. + * nxt_app_queue_cancel() (src/nxt_app_queue.h) is a CAS on the queue item's + * tracking word; nxt_unit_app_queue_recv() runs the same CAS from the worker + * side, so exactly one of the two wins and a false answer here means a worker + * claimed the slot first. + * + * The CAS runs at most once per request, and the answer is kept in + * ->msg_info.cancel for the callers that need it afterwards: both outcomes + * leave the tracking word at 0, so a second CAS would report a claim that + * never happened and clear ->is_port_mmap_sent on a chunk the worker owns. + */ + nxt_inline nxt_bool_t -nxt_router_msg_cancel(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) +nxt_router_msg_retract(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) { - nxt_buf_t *b, *next; - nxt_bool_t cancelled; nxt_port_t *app_port; nxt_msg_info_t *msg_info; msg_info = &req_rpc_data->msg_info; - if (msg_info->buf == NULL) { - return 0; - } + if (msg_info->cancel == NXT_MSG_QUEUED) { + app_port = req_rpc_data->app_port; - app_port = req_rpc_data->app_port; + if (app_port == NULL || app_port->id != NXT_SHARED_PORT_ID) { + /* Acknowledged: the message is the worker's, not the queue's. */ + return 0; + } - if (app_port != NULL && app_port->id == NXT_SHARED_PORT_ID) { - cancelled = nxt_app_queue_cancel(app_port->queue, - msg_info->tracking_cookie, - req_rpc_data->stream); + if (nxt_app_queue_cancel(app_port->queue, msg_info->tracking_cookie, + req_rpc_data->stream)) + { + msg_info->cancel = NXT_MSG_RETRACTED; - if (cancelled) { nxt_debug(task, "stream #%uD: cancelled by router", req_rpc_data->stream); + + } else { + msg_info->cancel = NXT_MSG_CLAIMED; + + nxt_debug(task, "stream #%uD: claimed by a worker", + req_rpc_data->stream); } + } - } else { - cancelled = 0; + return msg_info->cancel == NXT_MSG_RETRACTED; +} + + +nxt_inline nxt_bool_t +nxt_router_msg_cancel(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) +{ + nxt_buf_t *b, *next; + nxt_bool_t cancelled; + nxt_msg_info_t *msg_info; + + msg_info = &req_rpc_data->msg_info; + + if (msg_info->buf == NULL) { + return 0; } + cancelled = nxt_router_msg_retract(task, req_rpc_data); + for (b = msg_info->buf; b != NULL; b = next) { next = b->next; b->next = NULL; @@ -4130,6 +4162,19 @@ nxt_router_test_app_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_router_app_rpc_create(task, tmcf, app); } + +/* + * The request deadline, for src/test/nxt_router_app_timeout_test.c. That test + * drives it through the engine's timer machinery, so the wrapper is installed + * as the timer handler rather than called. + */ + +void +nxt_router_test_app_timeout(nxt_task_t *task, void *obj, void *data) +{ + nxt_router_app_timeout(task, obj, data); +} + #endif @@ -7080,6 +7125,32 @@ nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, req_rpc_data->app_port = port; req_rpc_data->apr_action = NXT_APR_REQUEST_FAILED; + /* + * Bound the wait for a process. Until a worker acknowledges the request + * the two dispatch paths that arm this timer have not run, so a request + * parked in ack_waiting_req has no deadline of its own: it waits for a + * worker that may never ask for it. With detached work that wait is as + * long as the application chooses to run, which is what makes the + * deadline worth having -- "limits": {"timeout"} now bounds waiting for + * capacity, not only the time a worker spends on the request. + * + * The handler is the one those paths use, and it is shared with the + * post-acknowledgement deadline, where 503 is simply the right answer. + * Here it is right only if the request is still in the queue, so the + * handler retracts it first and tells the two deadlines apart by the + * answer; see nxt_router_app_timeout(). + * + * An acknowledgement re-arms the same timer with the same handler, which + * nxt_timer_add() treats as a change, not a second timer. + */ + + if (app->timeout != 0) { + r->timer.handler = nxt_router_app_timeout; + r->timer_data = req_rpc_data; + + nxt_timer_add(task->thread->engine, &r->timer, app->timeout); + } + if (start_process) { nxt_router_start_app_process(task, app); @@ -7581,6 +7652,7 @@ static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) { nxt_timer_t *timer; + nxt_msg_info_t *msg_info; nxt_http_request_t *r; nxt_request_rpc_data_t *req_rpc_data; @@ -7591,6 +7663,45 @@ nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) r = nxt_timer_data(timer, nxt_http_request_t, timer); req_rpc_data = r->timer_data; + msg_info = &req_rpc_data->msg_info; + + /* + * Two deadlines share this handler. After an acknowledgement a worker + * holds the request and 503 is the answer: it is too slow. Before one + * the request is still in the shared port queue, and only the CAS in + * nxt_router_msg_retract() says whether it is still there. + * + * So retract before answering. A retraction that loses means a worker + * took the request in that same instant: it is running, and a 503 here + * would both fail the request and run it -- twice, if the client + * retries. + * + * Leave a claimed request alone until its acknowledgement, and do not + * re-arm: the acknowledgement arms this timer again. Do not answer it + * at a later expiry either. The chain's tail is the request body that + * nxt_router_req_headers_ack_handler() still has to send, and libunit + * keeps the request until that body arrives. An unlink here would drop + * the body and cancel the RPC, so a late acknowledgement is ignored and + * the request stays in the worker for ever. + * + * The acknowledgement replaces ->app_port with the worker's own port, so + * the shared port here means that it has not arrived yet. + */ + + if (msg_info->cancel == NXT_MSG_QUEUED) { + (void) nxt_router_msg_retract(task, req_rpc_data); + } + + if (msg_info->cancel == NXT_MSG_CLAIMED + && req_rpc_data->app_port != NULL + && req_rpc_data->app_port->id == NXT_SHARED_PORT_ID) + { + nxt_debug(task, "stream #%uD: claimed, waiting for the ack", + req_rpc_data->stream); + + return; + } + nxt_http_request_error(task, r, NXT_HTTP_SERVICE_UNAVAILABLE); nxt_request_rpc_data_unlink(task, req_rpc_data); diff --git a/src/nxt_router.h b/src/nxt_router.h index 8c3bdf3a4..90ebb568c 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -28,6 +28,14 @@ typedef struct nxt_http_comp_conf_s nxt_http_comp_conf_t; #define NXT_HTTP_ACTION_ERROR ((nxt_http_action_t *) -1) +/* + * The id of an application's shared port: the one whose queue every worker of + * the application reads, as opposed to a single worker's own port. A request + * message still sitting in that queue can be retracted; one sent to a worker + * port cannot. + */ +#define NXT_SHARED_PORT_ID 0xFFFFu + typedef struct { nxt_thread_spinlock_t lock; @@ -367,6 +375,9 @@ void nxt_router_test_detached_handler(nxt_task_t *task, nxt_router_temp_conf_t *nxt_router_test_temp_conf(nxt_task_t *task); void nxt_router_test_app_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_app_t *app); + +/* The request deadline handler, for the app-timeout test. */ +void nxt_router_test_app_timeout(nxt_task_t *task, void *obj, void *data); #endif diff --git a/src/nxt_router_request.h b/src/nxt_router_request.h index 6e226bae8..4d5f826ba 100644 --- a/src/nxt_router_request.h +++ b/src/nxt_router_request.h @@ -7,10 +7,27 @@ #define _NXT_ROUTER_REQUEST_H_INCLUDED_ +typedef enum { + /* Not offered to nxt_app_queue_cancel() yet. */ + NXT_MSG_QUEUED = 0, + /* The router took the message back; no worker will ever see it. */ + NXT_MSG_RETRACTED, + /* A worker claimed the queue slot first; the request is running. */ + NXT_MSG_CLAIMED, +} nxt_msg_cancel_t; + + typedef struct { nxt_buf_t *buf; nxt_fd_t body_fd; uint32_t tracking_cookie; + + /* + * What nxt_app_queue_cancel() answered, kept because the CAS cannot be + * asked twice: a win and a loss both leave the queue item's tracking + * word at 0, so a second call would report a claim that never happened. + */ + nxt_msg_cancel_t cancel; } nxt_msg_info_t; diff --git a/src/test/nxt_router_app_timeout_test.c b/src/test/nxt_router_app_timeout_test.c new file mode 100644 index 000000000..474d29331 --- /dev/null +++ b/src/test/nxt_router_app_timeout_test.c @@ -0,0 +1,546 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the request deadline answering a request a worker is + * already running. + * + * nxt_router_app_timeout() serves two deadlines. After an acknowledgement a + * worker holds the request and 503 is the answer. Before one the request is + * still in the application's shared port queue, and whether it can be taken + * back is decided by a single CAS: nxt_app_queue_cancel() and the worker's + * nxt_unit_app_queue_recv() run the same compare-and-swap on the queue item's + * tracking word, so exactly one of them wins. The handler used to answer 503 + * and unlink regardless of who won, which failed a request that was executing + * -- and ran it a second time if the client retried. + * + * Nothing about that window can be reached by racing a real application: it is + * one CAS wide. So the CAS is driven here instead. The "worker" is this test + * calling nxt_app_queue_recv() and nxt_app_queue_cancel() exactly as + * nxt_unit_app_queue_recv() (src/nxt_unit.c) does, before or after the + * deadline fires, and the handler runs through the engine's real timer + * machinery. + * + * Three things are pinned down: + * + * - a claimed request is not answered and its message is left intact, since + * the tail of that chain is the body the acknowledgement still has to + * send; + * - a later expiry does not answer it either: until the acknowledgement + * arrives, an unlink would drop that body and cancel the RPC, and libunit + * would keep the request for ever; + * - a request nobody claimed is still answered at once, and its retraction + * really takes the message out of the queue. + * + * The request is a hand-built nxt_http_request_t with ->header_sent set and no + * protocol: nxt_http_request_error() then stops at its own guard and reduces + * to nxt_http_request_error_handler(), which sets ->error and nothing else. + * That makes ->error the mark of "this expiry answered the request" without + * dragging a connection into the fixture. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +/* File scope: the deadline hands the application to the router's + reference-counting helpers, so its storage must outlive the frame. */ +static nxt_app_t nxt_router_app_timeout_test_app; + +static nxt_uint_t nxt_router_app_timeout_test_completions; + + +typedef struct { + nxt_http_request_t *r; + nxt_request_rpc_data_t *req_rpc_data; + nxt_buf_t *header; + nxt_buf_t *body; +} nxt_router_app_timeout_test_req_t; + + +static void +nxt_router_app_timeout_test_completion(nxt_task_t *task, void *obj, void *data) +{ + nxt_router_app_timeout_test_completions++; +} + + +/* + * Commit the pending timer changes and run what is due, the way + * nxt_event_engine_start() does. nxt_timer_find() folds the change list into + * the rbtree, nxt_timer_expire() queues the handlers that are due, and the + * drain runs them -- including a handler that re-arms its own timer, which is + * what the claimed request does. + */ + +static void +nxt_router_app_timeout_test_tick(nxt_task_t *task, nxt_event_engine_t *engine, + nxt_msec_t advance) +{ + nxt_task_t *wq_task; + nxt_work_handler_t handler; + void *obj, *data; + + (void) nxt_timer_find(engine); + + nxt_timer_expire(engine, engine->timers.now + advance); + + /* nxt_work_queue_pop() dereferences the head unconditionally. */ + + while (engine->fast_work_queue.head != NULL) { + handler = nxt_work_queue_pop(&engine->fast_work_queue, &wq_task, &obj, + &data); + + if (handler == NULL) { + continue; + } + + handler(wq_task != NULL ? wq_task : task, obj, data); + } +} + + +/* + * What a worker does with a queue item, from nxt_unit_app_queue_recv(): + * dequeue it, then claim it with the same CAS the router's retraction uses. + * Answers whether this side won. + */ + +static nxt_bool_t +nxt_router_app_timeout_test_claim(nxt_app_queue_t *queue) +{ + uint32_t cookie; + ssize_t size; + uint8_t rbuf[NXT_APP_QUEUE_MSG_SIZE]; + nxt_port_msg_t *port_msg; + + size = nxt_app_queue_recv(queue, rbuf, &cookie); + + if (size < (ssize_t) sizeof(nxt_port_msg_t)) { + return 0; + } + + port_msg = (nxt_port_msg_t *) rbuf; + + return nxt_app_queue_cancel(queue, cookie, port_msg->stream); +} + + +/* + * A request in the state nxt_router_app_port_get() leaves behind: its message + * in the shared port queue, itself parked in ack_waiting_req with no worker, + * and the deadline armed. Each leg gets its own request, because the unlink + * that ends one leaves that request's timer armed for the pool release. + */ + +static nxt_int_t +nxt_router_app_timeout_test_arm(nxt_task_t *task, nxt_event_engine_t *engine, + nxt_mp_t *mp, nxt_app_t *app, uint32_t stream, + nxt_router_app_timeout_test_req_t *t) +{ + int notify; + nxt_buf_t *header, *body; + nxt_http_request_t *r; + nxt_request_rpc_data_t *req_rpc_data; + + struct { + nxt_port_msg_t pm; + nxt_port_mmap_msg_t mm; + } nxt_packed msg; + + r = nxt_mp_zalloc(mp, sizeof(nxt_http_request_t)); + req_rpc_data = nxt_mp_zalloc(mp, sizeof(nxt_request_rpc_data_t)); + header = nxt_mp_zalloc(mp, sizeof(nxt_buf_t)); + body = nxt_mp_zalloc(mp, sizeof(nxt_buf_t)); + + if (nxt_slow_path(r == NULL || req_rpc_data == NULL + || header == NULL || body == NULL)) + { + return NXT_ERROR; + } + + r->mem_pool = mp; + r->engine = engine; + + /* See the file comment: keeps the 503 out of the protocol layer. */ + r->header_sent = 1; + r->proto.any = NULL; + + r->timer.task = &engine->task; + r->timer.work_queue = &engine->fast_work_queue; + r->timer.log = engine->task.log; + r->timer.bias = NXT_TIMER_DEFAULT_BIAS; + r->timer.handler = nxt_router_test_app_timeout; + r->timer_data = req_rpc_data; + + r->req_rpc_data = req_rpc_data; + + req_rpc_data->stream = stream; + req_rpc_data->app = app; + req_rpc_data->app_port = app->shared_port; + req_rpc_data->apr_action = NXT_APR_REQUEST_FAILED; + req_rpc_data->request = r; + req_rpc_data->msg_info.body_fd = -1; + + /* The RPC registry holds nothing for this stream. */ + req_rpc_data->rpc_cancel = 0; + + nxt_port_inc_use(app->shared_port); + + /* + * The message chain nxt_router_app_prepare_request() leaves behind: the + * header buffer that went through the queue, marked sent, and the body + * behind it, which nxt_router_req_headers_ack_handler() sends only once a + * worker acknowledges. + */ + + header->completion_handler = nxt_router_app_timeout_test_completion; + header->is_port_mmap_sent = 1; + header->next = body; + + body->completion_handler = nxt_router_app_timeout_test_completion; + + req_rpc_data->msg_info.buf = header; + + nxt_memzero(&msg, sizeof(msg)); + msg.pm.stream = stream; + msg.pm.type = NXT_PORT_MSG_REQ_HEADERS; + msg.pm.mmap = 1; + + if (nxt_slow_path(nxt_app_queue_send(app->shared_port->queue, &msg, + sizeof(msg), stream, ¬ify, + &req_rpc_data->msg_info.tracking_cookie) != NXT_OK)) + { + return NXT_ERROR; + } + + nxt_thread_mutex_lock(&app->mutex); + + app->active_requests++; + nxt_queue_insert_tail(&app->ack_waiting_req, &r->app_link); + + nxt_thread_mutex_unlock(&app->mutex); + + nxt_mp_retain(mp); + + nxt_timer_add(engine, &r->timer, app->timeout); + + t->r = r; + t->req_rpc_data = req_rpc_data; + t->header = header; + t->body = body; + + return NXT_OK; +} + + +nxt_int_t +nxt_router_app_timeout_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp, *req_mp; + nxt_app_t *app; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *shared_port; + nxt_bool_t app_mutex, timers; + nxt_runtime_t *rt, *saved_rt; + nxt_app_queue_t *queue; + nxt_event_engine_t engine, *saved_engine; + nxt_router_app_timeout_test_req_t claimed, unclaimed; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router app timeout test started"); + + ret = NXT_ERROR; + shared_port = NULL; + req_mp = NULL; + app_mutex = 0; + timers = 0; + app = &nxt_router_app_timeout_test_app; + + task = thr->task; + task->thread = thr; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + queue = nxt_mp_zalloc(mp, sizeof(nxt_app_queue_t)); + if (nxt_slow_path(rt == NULL || queue == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + /* The deadline runs on the engine's own task, as it does in the router. */ + + engine.task.thread = thr; + engine.task.log = thr->log; + + /* + * Real timer machinery: the re-arm under test is an nxt_timer_add() from + * inside the timer's own handler, and a memzero'd nxt_timers_t would let a + * re-arm that never lands look correct. + */ + + if (nxt_slow_path(nxt_timers_init(&engine.timers, 64) != NXT_OK)) { + goto done; + } + + timers = 1; + + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* The port every worker of the application reads, and its queue. */ + + shared_port = nxt_port_new(task, NXT_SHARED_PORT_ID, nxt_pid, + NXT_PROCESS_APP); + if (nxt_slow_path(shared_port == NULL)) { + goto done; + } + + shared_port->pair[0] = -1; + shared_port->pair[1] = -1; + shared_port->socket.fd = -1; + shared_port->queue_fd = -1; + + nxt_app_queue_init(queue); + shared_port->queue = queue; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "app-timeout-test"); + + app->shared_port = shared_port; + app->timeout = 1000; + app->processes = 1; + + /* Never reached by the releases below, so the deadline cannot free it. */ + app->use_count = 8; + + /* + * The pool the requests live in. Retained beyond what the legs release, + * so nothing frees a request out from under the assertions that follow. + */ + + req_mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(req_mp == NULL)) { + goto done; + } + + nxt_mp_retain(req_mp); + nxt_mp_retain(req_mp); + + /* + * Leg 1: a worker claims the slot before the deadline fires. The + * retraction inside the handler must lose, and the handler must then leave + * the request alone. + */ + + if (nxt_slow_path(nxt_router_app_timeout_test_arm(task, &engine, req_mp, + app, 0x4321, &claimed) + != NXT_OK)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: could not arm a request"); + goto done; + } + + if (nxt_slow_path(!nxt_router_app_timeout_test_claim(queue))) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: the worker could not claim " + "the queued message"); + goto done; + } + + nxt_router_app_timeout_test_completions = 0; + + nxt_router_app_timeout_test_tick(task, &engine, app->timeout); + + if (claimed.req_rpc_data->msg_info.cancel != NXT_MSG_CLAIMED) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: the deadline did not see the " + "claim (cancel %d)", + (int) claimed.req_rpc_data->msg_info.cancel); + goto done; + } + + if (claimed.r->error != 0 || claimed.r->req_rpc_data == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: a claimed request was " + "answered (error %d, rpc data %p)", + (int) claimed.r->error, claimed.r->req_rpc_data); + goto done; + } + + if (claimed.req_rpc_data->msg_info.buf != claimed.header + || claimed.header->next != claimed.body + || nxt_router_app_timeout_test_completions != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: the message of a claimed " + "request was completed (%ui buffers), so the " + "acknowledgement has no body left to send", + nxt_router_app_timeout_test_completions); + goto done; + } + + if (claimed.r->timer.enabled) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: the deadline was re-armed " + "for a claimed request with no acknowledgement"); + goto done; + } + + /* + * Leg 2: the acknowledgement is late. Later expiries must not answer the + * request, because the acknowledgement still has to send the body. + */ + + nxt_router_app_timeout_test_tick(task, &engine, app->timeout); + nxt_router_app_timeout_test_tick(task, &engine, 2 * app->timeout); + + if (claimed.r->error != 0 || claimed.r->req_rpc_data == NULL + || claimed.req_rpc_data->msg_info.buf != claimed.header + || claimed.header->next != claimed.body + || nxt_router_app_timeout_test_completions != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: a claimed request was " + "answered before its acknowledgement (error %d, " + "%ui buffers)", + (int) claimed.r->error, + nxt_router_app_timeout_test_completions); + goto done; + } + + /* + * Leg 3: the control. Nobody claims this one, so the retraction wins and + * the deadline answers at once -- the behaviour the fix must not change. + */ + + if (nxt_slow_path(nxt_router_app_timeout_test_arm(task, &engine, req_mp, + app, 0x4322, &unclaimed) + != NXT_OK)) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: could not arm a request"); + goto done; + } + + nxt_router_app_timeout_test_completions = 0; + + nxt_router_app_timeout_test_tick(task, &engine, app->timeout); + + if (unclaimed.req_rpc_data->msg_info.cancel != NXT_MSG_RETRACTED + || unclaimed.r->error == 0 || unclaimed.r->req_rpc_data != NULL) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: an unclaimed request was not " + "answered (cancel %d, error %d)", + (int) unclaimed.req_rpc_data->msg_info.cancel, + (int) unclaimed.r->error); + goto done; + } + + if (nxt_router_app_timeout_test_completions != 2 + || unclaimed.header->is_port_mmap_sent != 0) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: a retracted message did not " + "give its shared memory back (%ui buffers, sent %d)", + nxt_router_app_timeout_test_completions, + (int) unclaimed.header->is_port_mmap_sent); + goto done; + } + + /* + * The retraction has to be real, not just reported: a worker reaching the + * item now must find it cancelled and skip it, which is what + * nxt_unit_app_queue_recv() does with a CAS that answers false. + */ + + if (nxt_router_app_timeout_test_claim(queue)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router app timeout test: a worker still claimed a " + "retracted message"); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router app timeout test passed"); + + ret = NXT_OK; + +done: + + if (shared_port != NULL) { + nxt_port_use(task, shared_port, -1); + } + + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + if (timers) { + nxt_free(engine.timers.changes); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_thread_mutex_destroy(&rt->processes_mutex); + + if (req_mp != NULL) { + nxt_mp_destroy(req_mp); + } + + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 1da41a362..59cfe443f 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -230,6 +230,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_app_timeout_test(thr) != NXT_OK) { + return 1; + } + if (nxt_router_remove_pid_soak_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 5de282a73..bc30cf439 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -79,6 +79,7 @@ nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_death_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_timeout_test(nxt_thread_t *thr); +nxt_int_t nxt_router_app_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_detached_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); diff --git a/test/test_php_detached_max.py b/test/test_php_detached_max.py index 7f8f7f121..b64d82aa8 100644 --- a/test/test_php_detached_max.py +++ b/test/test_php_detached_max.py @@ -246,7 +246,12 @@ def test_php_detached_parked_request_times_out(): assert resp['status'] == 503, f'parked request timed out, got {resp}' assert elapsed < timeout + 5, f'answered at the deadline, took {elapsed:.1f}s' - assert not os.path.exists(ran), 'the timed-out request never executed' + # The end-to-end half of the claim. What it cannot show is the race + # underneath it -- a worker claiming the slot in the same instant the + # timer fires -- because that window is a single CAS. That half is + # driven deterministically in src/test/nxt_router_app_timeout_test.c, + # where the test plays the worker and chooses the CAS outcome. + assert not os.path.exists(ran), 'the timed-out request did not execute' # And it must still not run once the detached worker becomes free. time.sleep(3) From 8fa5df357e8d4c972ff52a3cf87536703206388e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 15 Sep 2026 23:29:31 +0200 Subject: [PATCH 218/391] router: retain detached app through replacement dispatch A detached worker can hold the last application reference after configuration removal. Start any replacement before releasing that reference, so dispatch cannot access a freed application. Cover worker death with and without configuration removal and spare capacity. Update existing status expectations for the detached counter, and query /status once per iteration in the settle loop. Consolidate the spare-port rebalance logic shared between nxt_router_app_port_busy() and nxt_router_app_port_close() into nxt_router_app_spare_rebalance(). In nxt_router_adjust_idle_timer(), the release-build guard beside the assert takes a detached port out of the idle queue without reaping it; a bare continue would select the same link again and spin with app->mutex held. When reporting detached work finished fails, defer delivery retry to the read loop rather than tearing down inline, and close the worker only if persistent failure prevents router accounting from settling. Split ctx_impl->detached into an operational state enum (NXT_UNIT_DETACHED_NONE / RUNNING) and a dedicated detached_retries attempt counter. Add unit tests for libunit FINISH-send failure covering retry with deferred graceful QUIT honoured and persistent-failure give-up teardown outside request processing. Co-Authored-By: Claude Opus 4.6 Harness: Antigravity --- CHANGES | 3 +- docs/changes.xml | 3 +- src/nxt_router.c | 118 +++++++++------- src/nxt_unit.c | 213 +++++++++++++++++++++++++++-- src/nxt_unit.h | 14 ++ src/test/nxt_unit_port_recv_test.c | 167 ++++++++++++++++++++++ test/test_app_start_timeout.py | 1 + test/test_php_detached_max.py | 55 ++++++++ test/test_status.py | 1 + 9 files changed, 510 insertions(+), 65 deletions(-) diff --git a/CHANGES b/CHANGES index 685ce21bb..86ae71e7b 100644 --- a/CHANGES +++ b/CHANGES @@ -7,7 +7,8 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 and the router returned the worker's slot to "processes": {"max"} and let its idle timer reap a process that was still executing. Such a worker now reports itself busy until the work ends, so "max" bounds - live processes and "/status" reports them as "detached". + live processes and "/status" reports them as "detached"; + the bug had appeared in 1.29.0. *) Change: "limits": {"timeout"} now also bounds how long a request waits for an application process, not only how long a worker may spend on diff --git a/docs/changes.xml b/docs/changes.xml index e1bb596af..1cabe1ba5 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -17,7 +17,8 @@ does this whenever a script calls fastcgi_finish_request(): the response goes out, the script runs on, and the router returned the worker's slot to "processes": {"max"} and let its idle timer reap a process that was still executing. Such a worker now reports itself busy until the work ends, so "max" -bounds live processes and "/status" reports them as "detached". +bounds live processes and "/status" reports them as "detached"; +the bug had appeared in 1.29.0. diff --git a/src/nxt_router.c b/src/nxt_router.c index 471ada738..23dca82e7 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1187,6 +1187,10 @@ nxt_router_msg_retract(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) msg_info = &req_rpc_data->msg_info; + if (msg_info->buf == NULL) { + return 0; + } + if (msg_info->cancel == NXT_MSG_QUEUED) { app_port = req_rpc_data->app_port; @@ -6520,6 +6524,40 @@ nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app) } +/* + * Refill spare_ports from the tail of idle_ports when a port leaves + * spare_ports. Called with app->mutex held. + */ +static void +nxt_router_app_spare_rebalance(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port) +{ + nxt_port_t *idle_port; + nxt_queue_link_t *idle_lnk; + + /* Check port was in 'spare_ports' using idle_start field. */ + if (port->idle_start == 0 && app->idle_processes >= app->spare_processes) { + /* + * If there is a vacant space in spare ports, + * move the last idle to spare_ports. + */ + nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); + + idle_lnk = nxt_queue_last(&app->idle_ports); + idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); + nxt_queue_remove(idle_lnk); + + nxt_queue_insert_tail(&app->spare_ports, idle_lnk); + + idle_port->idle_start = 0; + + nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " + "to spare_ports", + &app->name, idle_port->pid, idle_port->id); + } +} + + /* * Take a port out of the idle economy. Called with app->mutex held; answers * whether the caller must start a replacement process once it has dropped the @@ -6541,9 +6579,7 @@ static nxt_bool_t nxt_router_app_port_busy(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, const char *reason) { - nxt_bool_t start_process; - nxt_port_t *idle_port; - nxt_queue_link_t *idle_lnk; + nxt_bool_t start_process; start_process = 0; @@ -6557,26 +6593,7 @@ nxt_router_app_port_busy(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, &app->name, port->pid, port->id, (port->idle_start ? "idle_ports" : "spare_ports"), reason); - /* Check port was in 'spare_ports' using idle_start field. */ - if (port->idle_start == 0 && app->idle_processes >= app->spare_processes) { - /* - * If there is a vacant space in spare ports, - * move the last idle to spare_ports. - */ - nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); - - idle_lnk = nxt_queue_last(&app->idle_ports); - idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); - nxt_queue_remove(idle_lnk); - - nxt_queue_insert_tail(&app->spare_ports, idle_lnk); - - idle_port->idle_start = 0; - - nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " - "to spare_ports", - &app->name, idle_port->pid, idle_port->id); - } + nxt_router_app_spare_rebalance(task, app, port); if (nxt_router_app_can_start(app) && nxt_router_app_need_start(app)) { app->pending_processes++; @@ -6731,10 +6748,8 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, void nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) { - nxt_app_t *app; - nxt_bool_t unchain, start_process, detached; - nxt_port_t *idle_port; - nxt_queue_link_t *idle_lnk; + nxt_app_t *app; + nxt_bool_t unchain, start_process, detached; app = port->app; @@ -6779,23 +6794,7 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) &app->name, port->pid, port->id, (port->idle_start ? "idle_ports" : "spare_ports")); - if (port->idle_start == 0 - && app->idle_processes >= app->spare_processes) - { - nxt_assert(!nxt_queue_is_empty(&app->idle_ports)); - - idle_lnk = nxt_queue_last(&app->idle_ports); - idle_port = nxt_queue_link_data(idle_lnk, nxt_port_t, idle_link); - nxt_queue_remove(idle_lnk); - - nxt_queue_insert_tail(&app->spare_ports, idle_lnk); - - idle_port->idle_start = 0; - - nxt_debug(task, "app '%V' move port %PI:%d from idle_ports " - "to spare_ports", - &app->name, idle_port->pid, idle_port->id); - } + nxt_router_app_spare_rebalance(task, app, port); } app->processes--; @@ -6831,15 +6830,19 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) nxt_port_use(task, port, -1); } - /* The reference nxt_router_detached_apply() took for this worker. */ + if (start_process) { + nxt_router_start_app_process(task, app); + } + + /* + * Keep the detached reference until the replacement start has taken its + * own. After configuration removal this can be the last reference, so + * dropping it earlier frees app before nxt_router_start_app_process(). + */ if (detached) { nxt_router_app_use(task, app, -1); } - - if (start_process) { - nxt_router_start_app_process(task, app); - } } @@ -6895,6 +6898,21 @@ nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, void *data) nxt_assert(port->detached == 0); + if (nxt_slow_path(port->detached != 0)) { + /* + * A release build. A bare continue selects the same link again + * and spins with app->mutex held. Take the port out of the idle + * queue only: the worker is alive, so it keeps ->app, stays in + * app->processes and gets no QUIT. nxt_router_app_port_idle() + * inserts it again when the detached work ends. + */ + + nxt_queue_remove(lnk); + lnk->next = NULL; + app->idle_processes--; + continue; + } + timeout = port->idle_start + app->idle_timeout; nxt_debug(task, "app '%V' pid %PI, start %M, timeout %M, threshold %M", @@ -7759,7 +7777,7 @@ nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) nxt_port_t *port; nxt_bool_t changed, start_process, adjust_idle_timer; nxt_runtime_t *rt; - nxt_atomic_uint_t c; + nxt_atomic_int_t c; rt = task->thread->runtime; diff --git a/src/nxt_unit.c b/src/nxt_unit.c index e50ef7f1d..1d5f6e258 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -78,6 +78,7 @@ static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx); static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq); static void nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx); +static int nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx); static int nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state); static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); @@ -392,6 +393,12 @@ struct nxt_unit_read_buf_s { }; +enum { + NXT_UNIT_DETACHED_NONE = 0, + NXT_UNIT_DETACHED_RUNNING = 1, +}; + + struct nxt_unit_ctx_impl_s { nxt_unit_ctx_t ctx; @@ -438,6 +445,7 @@ struct nxt_unit_ctx_impl_s { * busy, and holds off a graceful quit, until it does. */ uint8_t detached; /* 1 bit */ + uint8_t detached_retries; nxt_unit_mmap_buf_t ctx_buf[2]; nxt_unit_read_buf_t ctx_read_buf; @@ -820,7 +828,8 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, * poison pointer. */ - ctx_impl->detached = 0; + ctx_impl->detached = NXT_UNIT_DETACHED_NONE; + ctx_impl->detached_retries = 0; nxt_queue_init(&ctx_impl->free_req); nxt_queue_init(&ctx_impl->free_ws); @@ -3534,6 +3543,18 @@ nxt_unit_buf_read(nxt_unit_buf_t **b, uint64_t *len, void *dst, size_t size) * sockets race. The router looks the worker up by pid. */ +#if (NXT_TESTS) +static unsigned int nxt_unit_test_send_detached_failure_count; + + +void +nxt_unit_test_send_detached_failures(unsigned int failures) +{ + nxt_unit_test_send_detached_failure_count = failures; +} +#endif + + static int nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state) { @@ -3544,6 +3565,15 @@ nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state) uint8_t state; } m; +#if (NXT_TESTS) + if (nxt_slow_path(state == NXT_PORT_DETACHED_FINISH + && nxt_unit_test_send_detached_failure_count > 0)) + { + nxt_unit_test_send_detached_failure_count--; + return NXT_UNIT_ERROR; + } +#endif + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); if (nxt_slow_path(lib->router_port == NULL)) { @@ -3588,11 +3618,11 @@ nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc) * response. */ - if (ctx_impl->detached == 0) { + if (ctx_impl->detached == NXT_UNIT_DETACHED_NONE) { if (nxt_unit_send_detached(req->ctx, NXT_PORT_DETACHED_START) == NXT_UNIT_OK) { - ctx_impl->detached = 1; + ctx_impl->detached = NXT_UNIT_DETACHED_RUNNING; } else { nxt_unit_req_alert(req, "failed to report a detached response"); @@ -3613,31 +3643,170 @@ nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc) static void nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx) { + int rc; nxt_unit_ctx_impl_t *ctx_impl; ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - if (ctx_impl->detached == 0) { + if (ctx_impl->detached == NXT_UNIT_DETACHED_NONE) { return; } - ctx_impl->detached = 0; + rc = nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_FINISH); + if (nxt_fast_path(rc == NXT_UNIT_OK)) { + ctx_impl->detached = NXT_UNIT_DETACHED_NONE; + ctx_impl->detached_retries = 0; + + /* + * A graceful QUIT that arrived during the work was deferred by + * nxt_unit_quit() on this flag, the way one that arrives during a + * request is deferred on active_req. That one is retried when the + * request is released; this is the equivalent. + */ - (void) nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_FINISH); + if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) { + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); + } + + return; + } /* - * A graceful QUIT that arrived during the work was deferred by - * nxt_unit_quit() on this flag, the way one that arrives during a - * request is deferred on active_req. That one is retried when the - * request is released; this is the equivalent. + * The FINISH edge failed to deliver (e.g. transient router-port buffer + * backpressure). Keep ctx_impl->detached set so the worker does not + * drop into an inconsistent state, and retry from the read loop rather + * than quitting inline inside nxt_unit_process_ready_req(). */ + ctx_impl->detached_retries = 1; +} - if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) { - nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); + +static int +nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx) +{ + int res; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + if (nxt_fast_path(ctx_impl->detached_retries == 0)) { + return NXT_UNIT_OK; } + + res = nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_FINISH); + if (nxt_fast_path(res == NXT_UNIT_OK)) { + ctx_impl->detached = NXT_UNIT_DETACHED_NONE; + ctx_impl->detached_retries = 0; + + if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) { + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); + } + + return NXT_UNIT_OK; + } + + if (++ctx_impl->detached_retries > 10) { + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + + nxt_unit_alert(ctx, "failed to report detached finish, closing worker"); + nxt_unit_quit(&lib->main_ctx.ctx, NXT_QUIT_NORMAL); + return NXT_UNIT_ERROR; + } + + return NXT_UNIT_OK; +} + + +#if (NXT_TESTS) +uint8_t +nxt_unit_test_ctx_detached(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->detached; +} + + +uint8_t +nxt_unit_test_ctx_detached_retries(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->detached_retries; +} + + +void +nxt_unit_test_ctx_set_detached(nxt_unit_ctx_t *ctx, uint8_t val) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + ctx_impl->detached = val; +} + + +void +nxt_unit_test_ctx_detached_done(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_detached_done(ctx); +} + + +int +nxt_unit_test_ctx_detached_retry(nxt_unit_ctx_t *ctx) +{ + return nxt_unit_ctx_detached_retry(ctx); +} + + +uint8_t +nxt_unit_test_ctx_online(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->online; +} + + +uint8_t +nxt_unit_test_ctx_ready(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->ready; } +void +nxt_unit_test_ctx_set_ready(nxt_unit_ctx_t *ctx, uint8_t val) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + ctx_impl->ready = val; +} + + +void +nxt_unit_test_ctx_quit_graceful(nxt_unit_ctx_t *ctx) +{ + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); +} +#endif + + void nxt_unit_request_done(nxt_unit_request_info_t *req, int rc) { @@ -5046,6 +5215,13 @@ nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + res = nxt_unit_ctx_detached_retry(ctx); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + return res; + } + } + if (ctx_impl->wait_items > 0 || !nxt_unit_chk_ready(ctx)) { return nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); } @@ -5237,6 +5413,10 @@ nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) nxt_queue_each(req_impl, &ready_req, nxt_unit_request_info_impl_t, port_wait_link) { + if (nxt_slow_path(!ctx_impl->online)) { + break; + } + lib = nxt_container_of(ctx_impl->ctx.unit, nxt_unit_impl_t, unit); req = &req_impl->req; @@ -5297,6 +5477,13 @@ nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) break; } + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + rc = nxt_unit_ctx_detached_retry(ctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + break; + } + } + retry: rc = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); @@ -6245,7 +6432,7 @@ nxt_unit_quit(nxt_unit_ctx_t *ctx, uint8_t quit_param) quit = nxt_queue_is_empty(&ctx_impl->active_req) && nxt_queue_is_empty(&ctx_impl->pending_rbuf) && ctx_impl->wait_items == 0 - && ctx_impl->detached == 0; + && ctx_impl->detached == NXT_UNIT_DETACHED_NONE; pthread_mutex_unlock(&ctx_impl->mutex); diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 35271acf8..9eaa915d3 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -412,4 +412,18 @@ void nxt_unit_req_log(nxt_unit_request_info_t *req, int level, nxt_unit_req_log(req, NXT_UNIT_LOG_ALERT, fmt, ##ARGS) +#if (NXT_TESTS) +void nxt_unit_test_send_detached_failures(unsigned int failures); +uint8_t nxt_unit_test_ctx_detached(nxt_unit_ctx_t *ctx); +uint8_t nxt_unit_test_ctx_detached_retries(nxt_unit_ctx_t *ctx); +void nxt_unit_test_ctx_set_detached(nxt_unit_ctx_t *ctx, uint8_t val); +void nxt_unit_test_ctx_detached_done(nxt_unit_ctx_t *ctx); +int nxt_unit_test_ctx_detached_retry(nxt_unit_ctx_t *ctx); +uint8_t nxt_unit_test_ctx_online(nxt_unit_ctx_t *ctx); +uint8_t nxt_unit_test_ctx_ready(nxt_unit_ctx_t *ctx); +void nxt_unit_test_ctx_set_ready(nxt_unit_ctx_t *ctx, uint8_t val); +void nxt_unit_test_ctx_quit_graceful(nxt_unit_ctx_t *ctx); +#endif + + #endif /* _NXT_UNIT_H_INCLUDED_ */ diff --git a/src/test/nxt_unit_port_recv_test.c b/src/test/nxt_unit_port_recv_test.c index cf05111f4..d4973a08f 100644 --- a/src/test/nxt_unit_port_recv_test.c +++ b/src/test/nxt_unit_port_recv_test.c @@ -43,6 +43,7 @@ #include #include #include +#include #if (NXT_HAVE_MEMFD_CREATE && NXT_HAVE_LINUX_MEMFD_H) #include @@ -249,6 +250,167 @@ nxt_port_recv_test_notify(void) } +#if (NXT_TESTS) +static void +nxt_port_recv_test_detached_single_fail(nxt_unit_ctx_t *ctx) +{ + int rc; + + /* + * Case A: FINISH send fails once, next retry in read loop succeeds, + * detached flag clears to 0, and worker keeps serving. + */ + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_send_detached_failures(0); + nxt_unit_test_ctx_set_detached(ctx, 1); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 0, + "detached initial state is running"); + + /* Inject single FINISH send failure. */ + nxt_unit_test_send_detached_failures(1); + + /* Simulate request handler return calling nxt_unit_ctx_detached_done. */ + nxt_unit_test_ctx_detached_done(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 1, + "failed finish send schedules retry"); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_online(ctx) == 1 + && nxt_unit_test_ctx_ready(ctx) == 1, + "worker remains online and ready during retry"); + + /* Simulate read loop retry. Subsequent send succeeds. */ + rc = nxt_unit_test_ctx_detached_retry(ctx); + + nxt_port_recv_test_assert( + rc == NXT_UNIT_OK + && nxt_unit_test_ctx_detached(ctx) == 0 + && nxt_unit_test_ctx_detached_retries(ctx) == 0 + && nxt_unit_test_ctx_online(ctx) == 1, + "retry settles detached = 0 and keeps serving"); + + /* + * Case B: Graceful QUIT arrives while detached work is running. + * Quit is held off because detached == 1, and only honoured once + * the retry succeeds. + */ + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_send_detached_failures(0); + nxt_unit_test_ctx_set_detached(ctx, 1); + + /* Graceful quit arrives while worker is detached. */ + nxt_unit_test_ctx_quit_graceful(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_ready(ctx) == 0 + && nxt_unit_test_ctx_online(ctx) == 1, + "graceful quit deferred while detached is running"); + + /* Inject single FINISH send failure on request handler return. */ + nxt_unit_test_send_detached_failures(1); + nxt_unit_test_ctx_detached_done(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 1 + && nxt_unit_test_ctx_online(ctx) == 1, + "graceful quit remains deferred while finish retry is pending"); + + /* + * Retry succeeds: detached clears and the deferred graceful quit is + * honoured. + */ + rc = nxt_unit_test_ctx_detached_retry(ctx); + + nxt_port_recv_test_assert( + rc == NXT_UNIT_OK + && nxt_unit_test_ctx_detached(ctx) == 0 + && nxt_unit_test_ctx_detached_retries(ctx) == 0 + && nxt_unit_test_ctx_online(ctx) == 0, + "retry settles detached = 0 and honours deferred graceful quit"); +} + + +static void +nxt_port_recv_test_detached_persistent_fail(nxt_unit_ctx_t *ctx) +{ + int i, rc, status; + pid_t pid; + + /* + * Persistent send failures (>10 retries): + * Test in a child process so that the worker teardown (via + * nxt_unit_quit(..., NXT_QUIT_NORMAL)) does not affect the main test. + */ + pid = fork(); + if (pid == -1) { + perror("fork"); + nxt_port_recv_test_failures++; + return; + } + + if (pid == 0) { + nxt_unit_test_ctx_set_detached(ctx, 1); + + /* Inject persistent failures for all attempts. */ + nxt_unit_test_send_detached_failures(20); + + /* Request handler return fails to send FINISH. */ + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_retries(ctx) != 1) + { + _exit(1); + } + + /* Retries 1..9 (attempts 2..10) return OK and increment retries. */ + for (i = 1; i <= 9; i++) { + rc = nxt_unit_test_ctx_detached_retry(ctx); + if (rc != NXT_UNIT_OK + || nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_retries(ctx) != (i + 1) + || nxt_unit_test_ctx_online(ctx) != 1) + { + _exit(2); + } + } + + /* + * Retry 10 (attempt 11): retries exceed 10. + * Give-up path executes outside request processing: + * worker is closed (online == 0) and returns NXT_UNIT_ERROR. + */ + rc = nxt_unit_test_ctx_detached_retry(ctx); + if (rc != NXT_UNIT_ERROR + || nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 11) + { + _exit(3); + } + + _exit(0); + } + + if (waitpid(pid, &status, 0) == -1) { + perror("waitpid"); + nxt_port_recv_test_failures++; + return; + } + + nxt_port_recv_test_assert( + WIFEXITED(status) && WEXITSTATUS(status) == 0, + "persistent send failures trigger give-up teardown outside request"); +} +#endif + + int main(void) { @@ -379,6 +541,11 @@ main(void) fcntl(unrelated, F_GETFD) != -1, "unrelated fd survives an empty read"); +#if (NXT_TESTS) + nxt_port_recv_test_detached_persistent_fail(ctx); + nxt_port_recv_test_detached_single_fail(ctx); +#endif + if (nxt_port_recv_test_failures != 0) { printf("port_recv test: %d failure(s)\n", nxt_port_recv_test_failures); diff --git a/test/test_app_start_timeout.py b/test/test_app_start_timeout.py index 46067ff52..3fb826cc8 100644 --- a/test/test_app_start_timeout.py +++ b/test/test_app_start_timeout.py @@ -611,6 +611,7 @@ def test_app_start_timeout_slow_worker_adopted(): 'starting': 0, 'unaccounted': 0, 'idle': 1, + 'detached': 0, } assert client.get(url='/')['status'] == 200, 'the adopted worker serves' diff --git a/test/test_php_detached_max.py b/test/test_php_detached_max.py index b64d82aa8..2cffbaecb 100644 --- a/test/test_php_detached_max.py +++ b/test/test_php_detached_max.py @@ -11,10 +11,13 @@ """ import os +import signal import subprocess import threading import time +import pytest + from unit.applications.lang.php import ApplicationPHP from unit.control import Control from unit.option import option @@ -446,3 +449,55 @@ def test_php_detached_survives_configuration_removal(): time.sleep(0.2) assert os.path.exists(done), 'the detached work finished after removal' + + +@pytest.mark.parametrize('spare', [0, 1]) +@pytest.mark.parametrize('remove', [False, True]) +def test_php_detached_worker_dies(skip_alert, spare, remove): + """A detached worker's last app reference survives replacement dispatch.""" + + client.load('detached_worker', processes={'max': 1, 'spare': spare}) + assert client.get(url='/?sleep=30')['status'] == 200 + + deadline = time.monotonic() + 10 + while app_processes().get('detached', 0) != 1: + assert time.monotonic() < deadline, 'the worker reported detached' + time.sleep(0.05) + + pids = worker_pids() + assert len(pids) == 1 + pid = int(pids.pop()) + skip_alert(fr'app process {pid} exited on signal 9') + + if remove: + assert 'success' in client.conf( + {'listeners': {}, 'applications': {}}, '/config' + ) + + os.kill(pid, signal.SIGKILL) + + deadline = time.monotonic() + 10 + while True: + status = Control().conf_get('/status') + apps = status.get('applications', {}) + procs = apps.get('detached_worker', {}).get('processes', {}) + if remove: + settled = 'detached_worker' not in apps and not worker_pids() + else: + settled = ( + procs.get('detached') == 0 and str(pid) not in worker_pids() + ) + if settled: + break + assert time.monotonic() < deadline, 'the dead worker was settled' + time.sleep(0.05) + + assert 'applications' in status + if remove: + assert 'detached_worker' not in status['applications'] + else: + assert client.get()['status'] == 200, 'replacement serves requests' + assert drain(), 'the replacement finished' + + # The standard fixture also checks that the router PID did not change: + # a restarted router answering /status must not hide a crash here. diff --git a/test/test_status.py b/test/test_status.py index 7e30c9541..b6ef7ae3e 100644 --- a/test/test_status.py +++ b/test/test_status.py @@ -197,6 +197,7 @@ def check_application(name, running, starting, idle, active): 'starting': starting, 'unaccounted': 0, 'idle': idle, + 'detached': 0, }, 'requests': {'active': active}, } From 7545594f1c4533f018d3df88e20f1e4d8b59a7e1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 16 Sep 2026 16:44:20 +0200 Subject: [PATCH 219/391] unit: retry detached lifecycle notifications Reject invalid detached state bytes at the router. Retry the START edge inside nxt_unit_request_done_detached(): the application continues inside its request handler, so the read loop cannot retry in time. 8 attempts with 1..64 ms sleeps, 127 ms in total. If all fail, do not mark the context detached, because the router does not know about the work, and quit the worker gracefully when the handler returns. While a FINISH retry is pending, bound the blocking waits of the read loop: poll() in nxt_unit_read_buf() and the read in nxt_unit_ctx_port_recv(). The wait starts at 2 ms and doubles up to 256 ms, so an idle worker reaches the give-up in about 0.8 s. Previously the retry waited for traffic. A request handler that returns while the FINISH is pending no longer resets the retry budget. In nxt_unit_run_ctx(), run the retry before taking a read buffer, so the give-up does not leak it. A retry that delivers the FINISH edge completes a graceful QUIT that was deferred on the detached flag, which removes the read port. Return from the read loop instead of receiving on that port: a receive there waits for a message the router will never send. nxt_unit_read_buf() reports "no message", so its caller releases the read buffer, and nxt_unit_run() maps that to NXT_UNIT_OK, the code it already returns for a QUIT message. Tests: the retry budget across a handler return, the START give-up, the FINISH give-up on an idle worker, ready and not ready, and both read loops returning on a retry that completes a deferred quit. Co-Authored-By: Claude Opus 5 --- src/nxt_router.c | 8 + src/nxt_unit.c | 277 +++++++++++++++++++++++++--- src/nxt_unit.h | 4 + src/test/nxt_unit_port_recv_test.c | 282 +++++++++++++++++++++++++++++ 4 files changed, 542 insertions(+), 29 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 23dca82e7..d5ba853c3 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -7955,6 +7955,14 @@ nxt_router_detached_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } + if (*msg->buf->mem.pos != NXT_PORT_DETACHED_START + && *msg->buf->mem.pos != NXT_PORT_DETACHED_FINISH) + { + nxt_alert(task, "detached_handler: invalid state byte %d", + (int) *msg->buf->mem.pos); + return; + } + nxt_router_detached_apply(task, msg->port_msg.pid, *msg->buf->mem.pos); } diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 1d5f6e258..0bc59462d 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -77,8 +77,11 @@ static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx); static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq); +static void nxt_unit_ctx_detached_start(nxt_unit_ctx_t *ctx); static void nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx); static int nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx); +static int nxt_unit_detached_timeout(nxt_unit_ctx_impl_t *ctx_impl); +static int nxt_unit_detached_poll(nxt_unit_ctx_t *ctx, int fd); static int nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state); static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); @@ -445,8 +448,16 @@ struct nxt_unit_ctx_impl_s { * busy, and holds off a graceful quit, until it does. */ uint8_t detached; /* 1 bit */ + + /* Failed FINISH sends so far; 0 when none is pending. */ uint8_t detached_retries; + /* + * The START edge was never delivered. The worker retires when the + * request handler returns. + */ + uint8_t detached_unreported; /* 1 bit */ + nxt_unit_mmap_buf_t ctx_buf[2]; nxt_unit_read_buf_t ctx_read_buf; @@ -830,6 +841,7 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, ctx_impl->detached = NXT_UNIT_DETACHED_NONE; ctx_impl->detached_retries = 0; + ctx_impl->detached_unreported = 0; nxt_queue_init(&ctx_impl->free_req); nxt_queue_init(&ctx_impl->free_ws); @@ -3566,9 +3578,7 @@ nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state) } m; #if (NXT_TESTS) - if (nxt_slow_path(state == NXT_PORT_DETACHED_FINISH - && nxt_unit_test_send_detached_failure_count > 0)) - { + if (nxt_slow_path(nxt_unit_test_send_detached_failure_count > 0)) { nxt_unit_test_send_detached_failure_count--; return NXT_UNIT_ERROR; } @@ -3601,10 +3611,6 @@ nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state) void nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc) { - nxt_unit_ctx_impl_t *ctx_impl; - - ctx_impl = nxt_container_of(req->ctx, nxt_unit_ctx_impl_t, ctx); - /* * Before the response, not after. The two go to different ports, so * this orders nothing on the router's side by itself -- the router @@ -3613,23 +3619,64 @@ nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc) * idle queue as short as the router's own read makes it. * * A failure here is not fatal to the request: the response still goes - * out below, and the worst case is the accounting this call was meant - * to correct, which is what the router does today for every such - * response. + * out below. */ - if (ctx_impl->detached == NXT_UNIT_DETACHED_NONE) { - if (nxt_unit_send_detached(req->ctx, NXT_PORT_DETACHED_START) + nxt_unit_ctx_detached_start(req->ctx); + + nxt_unit_request_done(req, rc); +} + + +/* + * Report the START edge. The application continues inside its request + * handler, and the read loop does not run again until the handler returns, + * so a retry from the read loop is too late. Retry here: 8 attempts, with + * sleeps of 1, 2, ... 64 ms between them, 127 ms in total. + * + * If all attempts fail, the router counts this worker idle while the + * application runs. Do not set ->detached, because the router does not + * know about the work. Retire the worker when the handler returns. + */ + +#define NXT_UNIT_DETACHED_START_ATTEMPTS 8 + +static void +nxt_unit_ctx_detached_start(nxt_unit_ctx_t *ctx) +{ + int i; + struct timespec ts; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + if (ctx_impl->detached != NXT_UNIT_DETACHED_NONE + || ctx_impl->detached_unreported) + { + return; + } + + for (i = 0; i < NXT_UNIT_DETACHED_START_ATTEMPTS; i++) { + + if (i > 0) { + ts.tv_sec = 0; + ts.tv_nsec = (1L << (i - 1)) * 1000000L; + + (void) nanosleep(&ts, NULL); + } + + if (nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_START) == NXT_UNIT_OK) { ctx_impl->detached = NXT_UNIT_DETACHED_RUNNING; - - } else { - nxt_unit_req_alert(req, "failed to report a detached response"); + return; } } - nxt_unit_request_done(req, rc); + ctx_impl->detached_unreported = 1; + + nxt_unit_alert(ctx, "failed to report a detached response, " + "retiring the worker when the request handler returns"); } @@ -3648,6 +3695,20 @@ nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx) ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + if (nxt_slow_path(ctx_impl->detached_unreported)) { + ctx_impl->detached_unreported = 0; + + /* + * The START edge was never delivered. Quit gracefully, the same + * way as the deferred quit below: the router then settles the + * process and starts a replacement if one is needed. + */ + + nxt_unit_quit(ctx, NXT_QUIT_GRACEFUL); + + return; + } + if (ctx_impl->detached == NXT_UNIT_DETACHED_NONE) { return; } @@ -3672,15 +3733,27 @@ nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx) } /* - * The FINISH edge failed to deliver (e.g. transient router-port buffer - * backpressure). Keep ctx_impl->detached set so the worker does not - * drop into an inconsistent state, and retry from the read loop rather - * than quitting inline inside nxt_unit_process_ready_req(). + * The FINISH edge was not delivered. Keep ->detached set and let the + * read loop retry; its blocking waits are bounded while a retry is + * pending (see nxt_unit_detached_timeout()). The give-up is not run + * here, inside nxt_unit_process_ready_req(). + * + * Only arm the retry if it is not armed. Otherwise each request handler + * that returns while the FINISH is pending resets the budget, and a + * worker that serves traffic never reaches the give-up. */ - ctx_impl->detached_retries = 1; + + if (ctx_impl->detached_retries == 0) { + ctx_impl->detached_retries = 1; + } } +/* + * Retry the FINISH edge. Called from the read loop, outside request + * processing. After 10 failed retries, close the worker. + */ + static int nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx) { @@ -3709,6 +3782,18 @@ nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx) if (++ctx_impl->detached_retries > 10) { lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + /* + * Close the main context: the router keeps the detached flag on the + * main port (id 0), and only closing that port settles the count + * and the application reference. This assumes one context, as in + * PHP, the only caller of nxt_unit_request_done_detached(). A + * non-main context closing alone would leave the worker flagged + * detached for the life of the process, and the main context is + * changed here without cross-thread synchronization. This is a + * comment and not nxt_assert(): libunit does not link the thread + * context that nxt_assert() needs. + */ + nxt_unit_alert(ctx, "failed to report detached finish, closing worker"); nxt_unit_quit(&lib->main_ctx.ctx, NXT_QUIT_NORMAL); return NXT_UNIT_ERROR; @@ -3718,6 +3803,48 @@ nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx) } +/* + * How long a read may block while a FINISH retry is pending: 2 ms after the + * first failed retry, doubling up to 256 ms. The ten retries then span + * about 0.8 s without traffic. + */ + +static int +nxt_unit_detached_timeout(nxt_unit_ctx_impl_t *ctx_impl) +{ + return 1 << nxt_min(ctx_impl->detached_retries - 1, 8); +} + + +/* + * Wait for "fd" to become readable while a FINISH retry is pending. + * Returns NXT_UNIT_AGAIN when the wait expires, so that the caller returns + * to the read loop, which retries. + */ + +static int +nxt_unit_detached_poll(nxt_unit_ctx_t *ctx, int fd) +{ + int nevents; + struct pollfd pfd; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + + nevents = poll(&pfd, 1, nxt_unit_detached_timeout(ctx_impl)); + + if (nevents == 0 || (nevents == -1 && errno == EINTR)) { + return NXT_UNIT_AGAIN; + } + + return NXT_UNIT_OK; +} + + #if (NXT_TESTS) uint8_t nxt_unit_test_ctx_detached(nxt_unit_ctx_t *ctx) @@ -3752,6 +3879,35 @@ nxt_unit_test_ctx_set_detached(nxt_unit_ctx_t *ctx, uint8_t val) } +void +nxt_unit_test_ctx_set_detached_retries(nxt_unit_ctx_t *ctx, uint8_t val) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + ctx_impl->detached_retries = val; +} + + +uint8_t +nxt_unit_test_ctx_detached_unreported(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->detached_unreported; +} + + +void +nxt_unit_test_ctx_detached_start(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_detached_start(ctx); +} + + void nxt_unit_test_ctx_detached_done(nxt_unit_ctx_t *ctx) { @@ -5146,6 +5302,17 @@ nxt_unit_run(nxt_unit_ctx_t *ctx) nxt_unit_quit(ctx, NXT_QUIT_NORMAL); break; } + + /* + * A read that returned no message because the context went offline + * ends the loop the same way a QUIT message does, so report it the + * same way: callers exit the process with this code. + */ + + if (nxt_slow_path(rc == NXT_UNIT_AGAIN && !ctx_impl->online)) { + rc = NXT_UNIT_OK; + break; + } } nxt_unit_ctx_release(ctx); @@ -5206,7 +5373,7 @@ nxt_unit_run_once_impl(nxt_unit_ctx_t *ctx) static int nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) { - int nevents, res, err; + int nevents, res, err, timeout; nxt_uint_t nfds; nxt_unit_impl_t *lib; nxt_unit_ctx_impl_t *ctx_impl; @@ -5215,11 +5382,31 @@ nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + timeout = -1; + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { res = nxt_unit_ctx_detached_retry(ctx); if (nxt_slow_path(res != NXT_UNIT_OK)) { return res; } + + if (nxt_slow_path(!ctx_impl->online)) { + /* + * The retry completed a graceful quit that was deferred on the + * detached flag, so the read port is removed. Report "no + * message": the caller releases the buffer and its loop stops + * on ->online. A receive here would wait for a message the + * router will never send. + */ + + rbuf->size = -1; + + return NXT_UNIT_AGAIN; + } + + if (ctx_impl->detached_retries > 0) { + timeout = nxt_unit_detached_timeout(ctx_impl); + } } if (ctx_impl->wait_items > 0 || !nxt_unit_chk_ready(ctx)) { @@ -5276,7 +5463,15 @@ nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) fds[1].revents = 0; - nevents = poll(fds, nfds, -1); + nevents = poll(fds, nfds, timeout); + + if (nxt_slow_path(nevents == 0)) { + /* A pending FINISH retry bounded the wait; the caller retries. */ + rbuf->size = -1; + + return NXT_UNIT_AGAIN; + } + if (nxt_slow_path(nevents == -1)) { err = errno; @@ -5471,23 +5666,38 @@ nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) rc = NXT_UNIT_OK; while (nxt_fast_path(ctx_impl->online)) { - rbuf = nxt_unit_read_buf_get(ctx); - if (nxt_slow_path(rbuf == NULL)) { - rc = NXT_UNIT_ERROR; - break; - } - if (nxt_slow_path(ctx_impl->detached_retries > 0)) { rc = nxt_unit_ctx_detached_retry(ctx); if (nxt_slow_path(rc != NXT_UNIT_OK)) { break; } + + /* + * The retry may have completed a deferred graceful quit, which + * removes the read port. Leave with the retry's NXT_UNIT_OK, + * before a buffer is taken; "rc" is that value. + */ + + if (nxt_slow_path(!ctx_impl->online)) { + break; + } + } + + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + rc = NXT_UNIT_ERROR; + break; } retry: rc = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); if (rc == NXT_UNIT_AGAIN) { + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + nxt_unit_read_buf_release(ctx, rbuf); + continue; + } + goto retry; } @@ -6725,8 +6935,10 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf) { int res, read; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_port_impl_t *port_impl; + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); read = 0; @@ -6783,6 +6995,13 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, return NXT_UNIT_AGAIN; } + if (nxt_slow_path(ctx_impl->detached_retries > 0 && port->in_fd != -1)) { + res = nxt_unit_detached_poll(ctx, port->in_fd); + if (res != NXT_UNIT_OK) { + return res; + } + } + res = nxt_unit_port_recv(ctx, port, rbuf); if (nxt_slow_path(res == NXT_UNIT_ERROR)) { return NXT_UNIT_ERROR; diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 9eaa915d3..2adb00802 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -417,6 +417,10 @@ void nxt_unit_test_send_detached_failures(unsigned int failures); uint8_t nxt_unit_test_ctx_detached(nxt_unit_ctx_t *ctx); uint8_t nxt_unit_test_ctx_detached_retries(nxt_unit_ctx_t *ctx); void nxt_unit_test_ctx_set_detached(nxt_unit_ctx_t *ctx, uint8_t val); +void nxt_unit_test_ctx_set_detached_retries(nxt_unit_ctx_t *ctx, + uint8_t val); +uint8_t nxt_unit_test_ctx_detached_unreported(nxt_unit_ctx_t *ctx); +void nxt_unit_test_ctx_detached_start(nxt_unit_ctx_t *ctx); void nxt_unit_test_ctx_detached_done(nxt_unit_ctx_t *ctx); int nxt_unit_test_ctx_detached_retry(nxt_unit_ctx_t *ctx); uint8_t nxt_unit_test_ctx_online(nxt_unit_ctx_t *ctx); diff --git a/src/test/nxt_unit_port_recv_test.c b/src/test/nxt_unit_port_recv_test.c index d4973a08f..5f2021923 100644 --- a/src/test/nxt_unit_port_recv_test.c +++ b/src/test/nxt_unit_port_recv_test.c @@ -52,6 +52,8 @@ static int nxt_port_recv_test_failures; static int nxt_port_recv_test_step; +static int nxt_port_recv_test_block; +static int nxt_port_recv_test_shared_queue_fd = -1; static int nxt_port_recv_test_sock_fd = -1; static int nxt_port_recv_test_queue_fd = -1; static int nxt_port_recv_test_ctx_queue_fd = -1; @@ -137,6 +139,13 @@ nxt_port_recv_test_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *buf, int fds[2]; struct cmsghdr *cmsg; + if (nxt_port_recv_test_block) { + /* A real blocking read, as libunit does without this callback. */ + *oob_size = 0; + + return read(port->in_fd, buf, buf_size); + } + if (nxt_port_recv_test_step++ > 0) { return 0; } @@ -337,6 +346,263 @@ nxt_port_recv_test_detached_single_fail(nxt_unit_ctx_t *ctx) } +static void +nxt_port_recv_test_detached_retry_budget(nxt_unit_ctx_t *ctx) +{ + int rc; + + /* + * A request handler that returns while the FINISH edge is pending sends + * it again, but must not reset the retry budget. Otherwise a worker + * that serves traffic never reaches the give-up. + * + * Three injected failures: the handler return, one read-loop retry, and + * a second handler return that must leave the count at 2, not 1. The + * fourth attempt succeeds and clears both. + */ + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + nxt_unit_test_send_detached_failures(3); + + nxt_unit_test_ctx_detached_done(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 1, + "failed finish send arms the retry budget"); + + rc = nxt_unit_test_ctx_detached_retry(ctx); + + nxt_port_recv_test_assert( + rc == NXT_UNIT_OK + && nxt_unit_test_ctx_detached_retries(ctx) == 2, + "read loop spends one retry"); + + /* A request handler returns while the FINISH is still pending. */ + nxt_unit_test_ctx_detached_done(ctx); + + nxt_port_recv_test_assert( + nxt_unit_test_ctx_detached(ctx) == 1 + && nxt_unit_test_ctx_detached_retries(ctx) == 2, + "handler return does not reset the retry budget"); + + rc = nxt_unit_test_ctx_detached_retry(ctx); + + nxt_port_recv_test_assert( + rc == NXT_UNIT_OK + && nxt_unit_test_ctx_detached(ctx) == 0 + && nxt_unit_test_ctx_detached_retries(ctx) == 0 + && nxt_unit_test_ctx_online(ctx) == 1, + "the next successful retry clears the state"); +} + + +static void +nxt_port_recv_test_child_wait(pid_t pid, const char *name) +{ + int status; + + if (pid == -1) { + perror("fork"); + nxt_port_recv_test_failures++; + return; + } + + if (waitpid(pid, &status, 0) == -1) { + perror("waitpid"); + nxt_port_recv_test_failures++; + return; + } + + if (!WIFEXITED(status)) { + printf("port_recv test: child killed by signal %d\n", + WIFSIGNALED(status) ? WTERMSIG(status) : -1); + + } else if (WEXITSTATUS(status) != 0) { + printf("port_recv test: child exit %d\n", WEXITSTATUS(status)); + } + + nxt_port_recv_test_assert(WIFEXITED(status) && WEXITSTATUS(status) == 0, + name); +} + + +static void +nxt_port_recv_test_detached_start_give_up(nxt_unit_ctx_t *ctx) +{ + pid_t pid; + + /* + * The START edge is retried inside the request handler, because the read + * loop does not run until the handler returns. In a child process: the + * give-up quits the worker. + */ + pid = fork(); + + if (pid == 0) { + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 0); + + /* Seven failures: the eighth and last attempt succeeds. */ + nxt_unit_test_send_detached_failures(7); + nxt_unit_test_ctx_detached_start(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_unreported(ctx) != 0) + { + _exit(1); + } + + nxt_unit_test_ctx_set_detached(ctx, 0); + + /* Eight failures: the worker is not marked detached. */ + nxt_unit_test_send_detached_failures(8); + nxt_unit_test_ctx_detached_start(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 0 + || nxt_unit_test_ctx_detached_unreported(ctx) != 1 + || nxt_unit_test_ctx_online(ctx) != 1) + { + _exit(2); + } + + /* The handler returns: the worker retires, without a FINISH. */ + nxt_unit_test_send_detached_failures(1); + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached_unreported(ctx) != 0) + { + _exit(3); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, + "failed start send is retried in place, then retires the worker"); +} + + +static void +nxt_port_recv_test_detached_idle_retry(nxt_unit_ctx_t *ctx, uint8_t ready, + const char *name) +{ + pid_t pid; + nxt_app_queue_t *q; + + /* + * A FINISH retry must not wait for traffic. The read loop runs on an + * idle worker: its waits must expire, so the retries reach the give-up. + * Reads block, so an unbounded wait lasts until SIGALRM kills the child. + * A ready context waits in poll() on both ports; a context that is not + * ready waits on its read port only. + */ + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + /* An empty shared queue, as the router leaves it. */ + q = mmap(NULL, sizeof(nxt_app_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED, nxt_port_recv_test_shared_queue_fd, 0); + if (q == MAP_FAILED) { + _exit(4); + } + + nxt_app_queue_init(q); + + nxt_unit_test_ctx_set_ready(ctx, ready); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + nxt_unit_test_send_detached_failures(20); + + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached_retries(ctx) != 1) { + _exit(1); + } + + alarm(5); + + (void) nxt_unit_run(ctx); + + if (nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 11) + { + _exit(2); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_deferred_quit_read(nxt_unit_ctx_t *ctx, int run_ctx, + const char *name) +{ + int rc; + pid_t pid; + + /* + * A graceful QUIT that arrives while the worker is detached is deferred. + * The retry that delivers the FINISH edge then completes it, which takes + * the context offline and removes its read port. The read loop must + * return at once: a receive on the removed port waits for a message the + * router will never send, so an unbounded wait lasts until SIGALRM kills + * the child. Both read loops run the retry, so check both. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + nxt_unit_test_ctx_quit_graceful(ctx); + + if (nxt_unit_test_ctx_ready(ctx) != 0 + || nxt_unit_test_ctx_online(ctx) != 1) + { + _exit(1); + } + + /* One failure: the handler return arms the retry, the retry sends. */ + nxt_unit_test_send_detached_failures(1); + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_retries(ctx) != 1) + { + _exit(2); + } + + alarm(5); + + rc = run_ctx ? nxt_unit_run_ctx(ctx) : nxt_unit_run(ctx); + + if (rc != NXT_UNIT_OK + || nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 0) + { + _exit(3); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + static void nxt_port_recv_test_detached_persistent_fail(nxt_unit_ctx_t *ctx) { @@ -478,6 +744,8 @@ main(void) return 1; } + nxt_port_recv_test_shared_queue_fd = dup(init.shared_queue_fd); + ctx = nxt_unit_init(&init); if (ctx == NULL) { printf("port_recv test: nxt_unit_init() failed\n"); @@ -542,7 +810,21 @@ main(void) "unrelated fd survives an empty read"); #if (NXT_TESTS) + /* + * single_fail runs last: its Case B leaves the context offline, and + * every other case expects it online. + */ + nxt_port_recv_test_detached_retry_budget(ctx); + nxt_port_recv_test_detached_start_give_up(ctx); + nxt_port_recv_test_detached_idle_retry(ctx, 1, + "finish retry reaches the give-up, ready and idle"); + nxt_port_recv_test_detached_idle_retry(ctx, 0, + "finish retry reaches the give-up, not ready and idle"); nxt_port_recv_test_detached_persistent_fail(ctx); + nxt_port_recv_test_deferred_quit_read(ctx, 0, + "nxt_unit_run() returns on a retry that completes a quit"); + nxt_port_recv_test_deferred_quit_read(ctx, 1, + "nxt_unit_run_ctx() returns on a retry that completes a quit"); nxt_port_recv_test_detached_single_fail(ctx); #endif From 27f12bdadf28786a1afa96bf15f0cc361e5f40e3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 16 Sep 2026 23:31:43 +0200 Subject: [PATCH 220/391] router: keep a worker the deadline gave up on out of the idle economy "limits": {"timeout"} answers the client 503 and then releases the request's accounting. The worker is not told and keeps executing, so the release returns a slot that is not free: the port is parked as idle, the reaper QUITs a process in the middle of a request, and the next request can be counted against a worker that is busy. A request a worker is running is not the router's to release. The acknowledgement is what moves the request's port off the shared queue and onto the worker -- nxt_router_msg_retract() reads the same fact -- so when that has happened the deadline enters the state a detached worker already has instead of letting the release decide. port->detached is the term of the idle transition that keeps the port out of idle_ports, and nxt_router_app_port_busy() unwinds a port that is already there, both under app->mutex, as the acknowledgement does. The unwind is defensive: a port with a live request is not in an idle queue, so it normally finds nothing and starts nothing. detached_router counts every request the router gave up on rather than holding one port-wide flag: two concurrent requests can exceed the deadline on one worker, and with a flag the first answer cleared the port while the second was still running. The application's own reason moved to detached_app, so a FINISH edge cannot drop a request's reason and the two clears cannot drop each other. The state is settled by the worker's last message for the stream, which is already where the answer of a request nobody tracks is dropped, and by the port closing, which settles a detached worker today. Keeping the registration alive until then is what makes the first edge reachable: the unlink would cancel it, and the answer would be dropped by the port layer with nothing left to settle the port. The state is published to the main thread, which is where a port close settles it, so both application references and the port reference are taken before the critical section that publishes it -- the order nxt_router_detached_apply() already uses. A close arriving in the window between the unlock and the increments would otherwise drop a reference this call had not taken yet, and the increment that followed would resurrect an application whose free was already queued. An application START edge arriving in the meantime records its own reason, and its FINISH edge settles only that one, so a worker that answered and kept running is never marked free by the response to the request the router gave up on -- the case test_php_detached_start_after_the_worker_went_idle covers, which now finds the port already held out rather than parked. An upgraded stream is excluded. The 101 is a non-last message, so the deadline is armed again and an idle websocket reaches this handler, but its accounting was already returned by NXT_APR_UPGRADE: there is no slot being held, and the mark would report the worker "detached" for the life of the connection. The websocket state is the discriminator because it is assigned beside that release and nowhere else, so the two cannot drift apart; a handshake still in flight or one that failed does hold its accounting and is still abandoned. The action the upgrade leaves behind is not used here: it is a name a later change can rename without this condition noticing. The state lands on app_port->main_app_port: the object the acknowledgement and a release reach through the same field, while a detached edge finds it by pid, and a worker's ports share one main port. /status reports such a worker under "detached" while it runs, which is the router's own view of a worker it may not hand out; that is the one semantic extension here and it is deliberate. Nothing on the client's side changes: the 503 and its timing are what they were, exactly one answer is sent, and the request still cannot execute twice. Before the acknowledgement the router does not know which worker holds the request, so that window is unchanged: it is the one-CAS race the C test drives, where the request is either retracted or claimed by a worker that then has to answer, and where the deadline still bounds the wait. detached_app stays a flag, and libunit's detached state is per-context: with "threads" > 1 a worker emits START, START, FINISH, FINISH for two concurrent fastcgi_finish_request() calls, and the first FINISH ends the port's detached state while the second context still runs. That is the state #405 merged, not this change's, and a count would first have to show that libunit's retry of an undelivered edge cannot deliver a duplicate START, which a flag absorbs and a count would not. Test: test_node_websockets.py::test_node_websockets_timeout_does_not_detach upgrades a stream under "limits": {"timeout"} and waits past the deadline with no traffic in either direction, then asserts "detached": 0. Without the exclusion it reads "detached": 1 and conftest reports leaked descriptors, the port never being settled. Only the accounting is asserted: the handler also answers 503 and drops the stream, which is wrong for a websocket and is issue #422. Test: test_php_detached_max.py::test_php_timed_out_request_keeps_the_worker_busy holds the worker past the deadline and then answers normally, so no detached edge reports anything. It asserts the router's own counters -- "detached": 1 with "idle": 0 past idle_timeout while the script still runs, the "ran" marker appended exactly once, and the port settled when the answer arrives -- and it fails on the pre-fix head with running: 0 and detached: 0, the worker having been reaped mid-script. The observation window checks itself with a "done" marker, so a loaded runner fails loudly instead of reading /status after the script answered. The count itself has no test: a deterministic one needs either a concurrent ASGI fixture or a C-level case driving two abandons on one port, both of which need scaffolding this commit does not add. Co-Authored-By: DeepSeek V4.1 Flash Harness: DeepSeek Harness Co-Authored-By: Claude Opus 5 --- CHANGES | 5 +- docs/changes.xml | 5 +- src/nxt_port.h | 22 +++ src/nxt_router.c | 240 ++++++++++++++++++++++++++++- src/nxt_router_request.h | 11 ++ test/php/detached_worker/index.php | 38 ++++- test/test_node_websockets.py | 42 +++++ test/test_php_detached_max.py | 117 ++++++++++++-- 8 files changed, 452 insertions(+), 28 deletions(-) diff --git a/CHANGES b/CHANGES index 86ae71e7b..39edeb757 100644 --- a/CHANGES +++ b/CHANGES @@ -7,7 +7,10 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 and the router returned the worker's slot to "processes": {"max"} and let its idle timer reap a process that was still executing. Such a worker now reports itself busy until the work ends, so "max" bounds - live processes and "/status" reports them as "detached"; + live processes and "/status" reports them as "detached". The same + holds for a request whose "limits": {"timeout"} deadline has passed: + the worker still running it is no longer returned to the idle pool + either, and keeps counting against "max" until it answers or exits; the bug had appeared in 1.29.0. *) Change: "limits": {"timeout"} now also bounds how long a request waits diff --git a/docs/changes.xml b/docs/changes.xml index 1cabe1ba5..e04e79892 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -17,7 +17,10 @@ does this whenever a script calls fastcgi_finish_request(): the response goes out, the script runs on, and the router returned the worker's slot to "processes": {"max"} and let its idle timer reap a process that was still executing. Such a worker now reports itself busy until the work ends, so "max" -bounds live processes and "/status" reports them as "detached"; +bounds live processes and "/status" reports them as "detached". The same holds +for a request whose "limits": {"timeout"} deadline has passed: the worker still +running it is no longer returned to the idle pool either, and keeps counting +against "max" until it answers or exits; the bug had appeared in 1.29.0. diff --git a/src/nxt_port.h b/src/nxt_port.h index bf233b543..1acd514fd 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -347,6 +347,28 @@ struct nxt_port_s { * application reports the work finished. */ uint8_t detached; + + /* + * The application reported detached work of its own. Its FINISH edge is + * what ends that; until then the port stays out of the idle economy even + * with no request left. + */ + uint8_t detached_app; + + /* + * The router put the port in the detached state itself: one for each + * request it has given up on that the worker is still running. A + * "limits": {"timeout"} expiry answers the client while the worker keeps + * executing, and the port may not rejoin the idle economy until every + * such request has been answered or the port closes. A count rather + * than a flag because one worker can run several of them at once, and + * kept apart from detached_app so that neither clear drops the other's + * reason. As wide as active_requests below it, which counts the same + * population: "threads" is validated up to NXT_INT32_T_MAX, and a wrap + * would leave a settle unable to clear the state at all. + */ + uint32_t detached_router; + uint32_t active_requests; nxt_port_handler_t handler; diff --git a/src/nxt_router.c b/src/nxt_router.c index d5ba853c3..8bed7bb9b 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -368,6 +368,8 @@ static void nxt_router_app_unlink(nxt_task_t *task, nxt_app_t *app); static void nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_apr_action_t action); +static void nxt_router_app_abandoned_settle(nxt_task_t *task, + nxt_request_rpc_data_t *req_rpc_data); static void nxt_router_app_port_get(nxt_task_t *task, nxt_app_t *app, nxt_request_rpc_data_t *req_rpc_data); static void nxt_router_http_request_error(nxt_task_t *task, void *obj, @@ -5364,6 +5366,17 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, r = req_rpc_data->request; if (nxt_slow_path(r == NULL)) { + /* + * A request the router gave up on. The answer belongs to nobody and + * needs no reply, but the last message is the worker saying it is + * done: this is where the port it was running on may rejoin the idle + * economy. A message that is not the last one proves nothing yet; + * the response can still be streaming. + */ + if (msg->port_msg.last != 0) { + nxt_router_app_abandoned_settle(task, req_rpc_data); + } + return; } @@ -5810,6 +5823,12 @@ nxt_router_response_error_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, req_rpc_data->rpc_cancel = 0; + /* + * The worker is gone, so whatever the router gave up on it is over. The + * settle does nothing unless this request is one of those. + */ + nxt_router_app_abandoned_settle(task, req_rpc_data); + /* TODO cancel message and return if cancelled. */ if (req_rpc_data->request != NULL) { @@ -6658,6 +6677,167 @@ nxt_router_app_port_idle(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port) } +/* + * The router gave up on a request a worker is running. The 503 is the + * client's answer, but the worker does not know that and keeps executing, so + * the port must not go back into the idle economy when the request's + * accounting is released a moment from now: the reaper would QUIT a worker + * in the middle of a request, and the port would count as free against + * "processes": {"max"} while it is not. + * + * Enter the state a detached worker already has -- see + * nxt_router_detached_apply() -- and settle it when the worker answers, when + * the port closes, or when the application's own FINISH edge says its work is + * over. "app_port" is the port the acknowledgement moved the request to: + * the worker's, never the shared queue. Called before the unlink that + * releases the request. + * + * The state lands on app_port->main_app_port. That is the same object the + * acknowledgement and a release reach through the same field, while a + * detached edge finds it by pid (nxt_router_detached_apply()); a worker's + * ports share one main port, so both identities name it. + */ + +static void +nxt_router_app_abandon(nxt_task_t *task, nxt_app_t *app, + nxt_request_rpc_data_t *req_rpc_data, nxt_port_t *app_port) +{ + nxt_bool_t changed, start_process; + nxt_port_t *port; + + port = app_port->main_app_port; + + /* + * Both the detached state and the rpc_data's own pointer outlive this + * call, and the state is published to another thread: the port close + * path takes the same mutex and drops the state's reference when it + * settles. So both application references and the port reference are + * taken before the state can be seen, the way + * nxt_router_detached_apply() takes its own before the mutex; a close + * that arrived in between would otherwise drop a reference this call has + * not taken yet. The request still holds an application reference of + * its own here, so the increment cannot resurrect a freed one. + */ + nxt_router_app_use(task, app, 2); + + nxt_thread_mutex_lock(&app->mutex); + + changed = 0; + + if (port->detached == 0) { + port->detached = 1; + app->detached_processes++; + changed = 1; + } + + port->detached_router++; + + nxt_port_inc_use(port); + + /* + * Defensive, and symmetric with the detached START edge: a port with a + * live request is not in an idle queue, so this normally finds nothing + * to unwind and starts nothing. It is here so that the flag and the + * unwind share this critical section whichever way the port got here. + */ + start_process = nxt_router_app_port_busy(task, app, port, "abandoned"); + + nxt_thread_mutex_unlock(&app->mutex); + + if (!changed) { + /* + * The application was detached already, so its own edge owns that + * reference; this call keeps only the rpc_data's. + */ + nxt_router_app_use(task, app, -1); + } + + /* + * Keep the registration. nxt_router_response_ready_handler() is what + * drops the answer of a request nobody tracks any more, and that is also + * where the port is taken back; the unlink that follows would cancel the + * registration first and the answer would never reach it. + */ + req_rpc_data->rpc_cancel = 0; + + req_rpc_data->abandoned_app = app; + req_rpc_data->abandoned_port = port; + + if (start_process) { + nxt_router_start_app_process(task, app); + } +} + + +/* + * A worker the router gave up on has answered, or is gone. Its port rejoins + * the idle economy if the router's own mark is the only thing holding it out + * -- an application START edge clears that mark and its FINISH edge is what + * ends the state then -- and the references the abandonment took are dropped + * either way. + */ + +static void +nxt_router_app_abandoned_settle(nxt_task_t *task, + nxt_request_rpc_data_t *req_rpc_data) +{ + nxt_port_t *port; + nxt_app_t *app; + nxt_bool_t adjust_idle_timer, ours; + + port = req_rpc_data->abandoned_port; + + if (port == NULL) { + return; + } + + app = req_rpc_data->abandoned_app; + + req_rpc_data->abandoned_port = NULL; + req_rpc_data->abandoned_app = NULL; + + ours = 0; + adjust_idle_timer = 0; + + nxt_thread_mutex_lock(&app->mutex); + + if (port->detached_router != 0) { + port->detached_router--; + + /* + * The port is free only once no reason is left: this was the last + * request the router gave up on, and the application is not running + * detached work of its own either. + */ + if (port->detached_router == 0 && port->detached_app == 0 + && port->detached != 0) + { + port->detached = 0; + + app->detached_processes--; + + adjust_idle_timer = nxt_router_app_port_idle(task, app, port); + + ours = 1; + } + } + + nxt_thread_mutex_unlock(&app->mutex); + + if (adjust_idle_timer) { + nxt_router_app_use(task, app, 1); + nxt_event_engine_post(app->engine, &app->adjust_idle_work); + } + + if (ours) { + nxt_router_app_use(task, app, -1); + } + + nxt_router_app_use(task, app, -1); + nxt_port_use(task, port, -1); +} + + static void nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_apr_action_t action) @@ -6811,6 +6991,8 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) if (detached) { port->detached = 0; + port->detached_app = 0; + port->detached_router = 0; app->detached_processes--; } @@ -7720,6 +7902,32 @@ nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) return; } + /* + * The acknowledgement moves the request's port off the shared queue and + * onto the worker that took it; nxt_router_msg_retract() reads the same + * fact. A request a worker is running is not the router's to release: + * its port has to stay out of the idle economy until the worker answers + * or closes. See nxt_router_app_abandon(). + * + * An upgraded stream is not such a request. Its accounting was already + * given back by NXT_APR_UPGRADE, so there is no slot being held and + * nothing to keep out of the idle economy; the mark would only report the + * worker "detached" for as long as the connection lives. The websocket + * state is the upgrade itself -- it is assigned beside that release, and + * nowhere else -- so a handshake still in flight or one that failed, both + * of which do hold their accounting, still reach the abandon. + */ + if (req_rpc_data->app_port != NULL + && req_rpc_data->app_port->id != NXT_SHARED_PORT_ID + && r->state != &nxt_http_websocket) + { + /* The request is still linked, so it holds the application. */ + nxt_assert(req_rpc_data->app != NULL); + + nxt_router_app_abandon(task, req_rpc_data->app, req_rpc_data, + req_rpc_data->app_port); + } + nxt_http_request_error(task, r, NXT_HTTP_SERVICE_UNAVAILABLE); nxt_request_rpc_data_unlink(task, req_rpc_data); @@ -7836,6 +8044,12 @@ nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) if (state == NXT_PORT_DETACHED_START) { + /* + * A reason of the application's own, held beside the router's count + * of requests it gave up on: neither clear may drop the other. + */ + port->detached_app = 1; + if (port->detached == 0) { port->detached = 1; app->detached_processes++; @@ -7864,18 +8078,28 @@ nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) "detached"); } - } else if (port->detached != 0) { - port->detached = 0; - app->detached_processes--; - changed = 1; + } else { + port->detached_app = 0; /* - * Only now may this worker rejoin the idle economy. Its last - * response went out long ago, so nothing else will run this - * transition for it. + * The port leaves the detached state only when no reason is left: + * nothing here while a request the router gave up on is still + * running, and nothing for that request's answer while the + * application's own work is. */ + if (port->detached_router == 0 && port->detached != 0) { + port->detached = 0; + app->detached_processes--; + changed = 1; - adjust_idle_timer = nxt_router_app_port_idle(task, app, port); + /* + * Only now may this worker rejoin the idle economy. Its last + * response went out long ago, so nothing else will run this + * transition for it. + */ + + adjust_idle_timer = nxt_router_app_port_idle(task, app, port); + } } nxt_thread_mutex_unlock(&app->mutex); diff --git a/src/nxt_router_request.h b/src/nxt_router_request.h index 4d5f826ba..a36ef0e24 100644 --- a/src/nxt_router_request.h +++ b/src/nxt_router_request.h @@ -50,6 +50,17 @@ typedef struct { nxt_http_request_t *request; nxt_msg_info_t msg_info; + /* + * The worker's main port and its application, when the router gave up on + * a request that worker was running. Each is held by a reference of its + * own until the worker answers, because neither the request nor its + * accounting outlives the decision to stop waiting: the port stays out of + * the idle economy until then and the application has to stay alive for + * that state to be settled at all. + */ + nxt_port_t *abandoned_port; + nxt_app_t *abandoned_app; + nxt_bool_t rpc_cancel; } nxt_request_rpc_data_t; diff --git a/test/php/detached_worker/index.php b/test/php/detached_worker/index.php index 434fc672c..ac5980cb1 100644 --- a/test/php/detached_worker/index.php +++ b/test/php/detached_worker/index.php @@ -7,13 +7,21 @@ * * Query parameters, all optional: * - * ran=PATH write PATH before answering -- proves the request executed - * at all, which a request that was cancelled while parked - * must not do. + * ran=PATH append the worker pid to PATH before answering -- proves + * the request executed, and how many times: a request that + * was cancelled while parked must not execute at all. + * hold=N hold the worker for N seconds and then answer normally, + * with no fastcgi_finish_request(). The router's + * "limits": {"timeout"} deadline fires while the script still + * runs and nothing else ever reports that the worker is busy, + * so the router has to keep it out of the idle economy on its + * own until the answer arrives. * before=N hold the worker for N seconds *before* answering. Past - * "limits": {"timeout"} the router fails the request and - * parks the worker as idle, so the detached report that - * follows arrives for a port that is already in idle_ports. + * "limits": {"timeout"} the router fails the request while the + * script still runs, so the worker stays out of the idle + * economy for it and the detached report that follows is what + * ends that state -- unless the answer settles first, in which + * case the port is parked and the report takes it back out. * sleep=N hold the worker for N seconds after answering. * done=PATH write PATH after the sleep -- proves the detached work ran * to completion rather than being killed part-way. @@ -27,7 +35,23 @@ $ran = isset($_GET['ran']) ? (string) $_GET['ran'] : ''; if ($ran !== '') { - file_put_contents($ran, (string) getmypid()); + file_put_contents($ran, (string) getmypid() . "\n", FILE_APPEND); +} + +$hold = isset($_GET['hold']) ? (int) $_GET['hold'] : 0; + +if ($hold > 0) { + sleep($hold); + + $done = isset($_GET['done']) ? (string) $_GET['done'] : ''; + + if ($done !== '') { + file_put_contents($done, (string) getmypid()); + } + + echo "done"; + + return; } $before = isset($_GET['before']) ? (int) $_GET['before'] : 0; diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 44fd090f4..b4ebd5245 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -5,6 +5,7 @@ from unit.applications.lang.node import ApplicationNode from unit.applications.websockets import ApplicationWebsocket +from unit.control import Control prerequisites = {'modules': {'node': 'any'}} @@ -1432,3 +1433,44 @@ def test_node_websockets_keepalive_interval(): check_frame(frame, True, ws.OP_PING, '') # PING frame sock.close() + + +def test_node_websockets_timeout_does_not_detach(): + """An upgraded stream is not a request the deadline gave up on. + + "limits": {"timeout"} re-arms nxt_router_app_timeout() for every non-last + message from the application, and the 101 is one, so an idle websocket + reaches that handler with the deadline passed. The request's accounting + was already returned by the upgrade, so there is no worker still running + it: the router must not report the process "detached", which is its view + of a worker holding a request whose deadline passed. + + Only the accounting is asserted. The handler also answers 503 and drops + the stream, which is wrong for a websocket and is issue #422. + """ + + client.load('websockets/mirror') + + assert 'success' in client.conf( + {'timeout': 1}, 'applications/websockets%2Fmirror/limits' + ), 'configure timeout' + + _, sock, _ = ws.upgrade() + + # No traffic either way -- keepalive_interval is 0 from the fixture -- so + # nothing re-arms the timer and it expires. + time.sleep(3) + + processes = ( + Control() + .conf_get('/status') + .get('applications', {}) + .get('websockets/mirror', {}) + .get('processes', {}) + ) + + assert processes.get('detached', 0) == 0, ( + f'an upgraded stream marked the worker detached: {processes}' + ) + + sock.close() diff --git a/test/test_php_detached_max.py b/test/test_php_detached_max.py index 2cffbaecb..dd018e651 100644 --- a/test/test_php_detached_max.py +++ b/test/test_php_detached_max.py @@ -270,17 +270,27 @@ def test_php_detached_start_after_the_worker_went_idle(): The start edge is sent before the last response message, but that only orders the two on the wire. A request that is failed rather than - answered releases its worker with no start edge in sight at all: - "limits": {"timeout"} answers 503 while the script is still running, and - the release puts the port in idle_ports. The start edge then arrives for - a port that is already there. - - Unless it is taken back out, the reaper QUITs a process that is still - executing PHP -- the exact failure this feature exists to prevent -- and - clears the port's application on the way, after which nothing settles the - detached state: nxt_port_close() skips nxt_router_app_port_close() for a - port with no application, the finish edge is ignored for the same reason, - and the count sticks for the life of the application. + answered has no start edge in sight at all: "limits": {"timeout"} answers + 503 while the script is still running, and the port is held out of the + idle economy until the report arrives -- by the release, and by the + router's own mark once the acknowledgement has moved the request to the + worker. + + When the mark is in place, the edge finds the port already out and has + nothing to unwind; it records the application's own reason, and the + FINISH ends the state. The unwind is still load-bearing for the other + order: the answer's settle runs on the request engine and this edge on the + main thread, nothing orders their readers, and a settle that wins clears + the router's mark, lets the release park the port, and leaves + nxt_router_app_port_busy(..., "detached") to take it back out. Both + orders are reachable as this test runs; the assertions hold either way. + + Without that, the reaper QUITs a process that is still executing PHP -- + the exact failure this feature exists to prevent -- and clears the port's + application on the way, after which nothing settles the detached state: + nxt_port_close() skips nxt_router_app_port_close() for a port with no + application, the finish edge is ignored for the same reason, and the count + sticks for the life of the application. """ timeout = 2 @@ -346,6 +356,91 @@ def test_php_detached_start_after_the_worker_went_idle(): assert os.path.exists(done), 'the detached work ran to completion' +def test_php_timed_out_request_keeps_the_worker_busy(): + """A request the router gave up on must not free the worker running it. + + The script holds the worker past limits.timeout and then answers normally: + no fastcgi_finish_request(), so no detached edge ever reports that it is + still running. Nothing else tells the router either -- the deadline + answers 503 while the worker keeps executing -- so the router's own + accounting has to keep the port out of the idle economy until the answer + arrives. Released, the port is parked as idle, the reaper QUITs a worker + in the middle of a script, and the slot counts as free under "max": 1. + + The counters asserted are the router's own view of that: "detached", for a + worker it may not hand out, and "idle", which must stay empty. The "ran" + marker is appended once per execution, so it pins that the request ran + exactly once as well. + """ + + timeout = 2 + hold = 12 + + client.load( + 'detached_worker', + processes={'max': 1, 'spare': 0, 'idle_timeout': IDLE_TIMEOUT}, + limits={'timeout': timeout}, + ) + + ran = marker('timed_out_ran') + done = marker('timed_out_done') + + start = time.time() + resp = client.get( + url=f'/?ran={ran}&hold={hold}&done={done}', read_timeout=timeout + 10 + ) + elapsed = time.time() - start + + assert resp['status'] == 503, f'the request outlived limits.timeout: {resp}' + assert elapsed < timeout + 5, f'answered at the deadline, took {elapsed:.1f}s' + assert os.path.exists(ran), 'the request executed' + + # Well past idle_start + idle_timeout with the script still running: a + # port the router had released would have been reaped by now. The + # marker says the window really is open, so a slow machine fails here + # loudly instead of reading counters after the script has answered. + time.sleep(IDLE_TIMEOUT * 2) + + assert not os.path.exists(done), ( + f'the script finished within {IDLE_TIMEOUT * 2}s of the deadline; ' + f'raise hold above {hold}' + ) + + procs = app_processes() + + assert procs.get('detached', 0) == 1, ( + f'the worker running the failed request is not held out of the idle ' + f'economy: {procs}' + ) + assert procs.get('idle', 0) == 0, ( + f'a worker still running a request the router gave up on is counted ' + f'idle and is reapable: {procs}' + ) + assert procs.get('running', 0) == 1, ( + f'the worker was reaped while the script ran: {procs}' + ) + assert len(worker_pids()) == 1, '"max": 1 still bounds live workers' + + # The script answers at "hold"; the worker is free from that message on. + deadline = time.time() + hold + 20 + while time.time() < deadline and ( + app_processes().get('detached', 0) != 0 or not os.path.exists(done) + ): + time.sleep(0.2) + + assert os.path.exists(done), 'the script ran to completion' + assert app_processes().get('detached', 0) == 0, ( + f"the port was not settled by the worker's answer: {app_processes()}" + ) + + with open(ran) as f: + pids = f.read().split() + + assert len(pids) == 1, f'the request executed {len(pids)} times: {pids}' + + assert client.get()['status'] == 200, 'the worker serves again' + + def test_php_detached_repeated_requests_hold_the_bound(): """Repeated detached requests must not accumulate live workers. From c24e2d5b1c7eb8946d49a37853ee6e32ac3d14e3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 18 Sep 2026 12:20:10 +0200 Subject: [PATCH 221/391] router: settle the websocket count when the session ends A worker that serves a websocket upgrade is counted busy for the life of the session: nxt_router_app_port_release() counts the session on the worker's main port and nxt_router_app_port_idle() refuses to hand a port with active_websockets back to the idle economy. Nothing ever uncounted it, so the worker never came back: it stayed out of idle_ports and spare_ports, the reaper never saw it, and its slot kept counting against "processes": {"max"} until the process died. One upgrade was enough, and a long-lived application leaks a slot per upgrade until it can start no more workers. The decrement was lost in 83595606 ("Introducing the shared application port"). Before it the counter was a queue of the requests upgraded on the port, and nxt_request_rpc_data_unlink() removed the request from it along with the other per-request queues. That commit replaced the queue with a plain counter and deleted the whole nxt_request_app_link_t block the removal lived in, leaving the increment with no counterpart. The same unlink is still the right place, and it already releases the port: a websocket that ends reaches it whichever side ends it -- the application's last response message, an error from the worker, the death of the worker, or the close of the client connection, which asks the worker for the last frame and is answered by that same response. So the action the upgrade leaves behind for it now says what it is. NXT_APR_WEBSOCKET_CLOSE drops the reference the way NXT_APR_CLOSE did and uncounts the session under the application mutex, next to the request accounting it mirrors. nxt_router_app_timeout() reaches that unlink too, for an upgraded stream whose "limits": {"timeout"} passed, and the action carries the uncount there as well: it is the one thing that runs for such a session, since the same handler drops the stream and cancels the RPC, so no later message from the worker unlinks it a second time. A counter rather than a flag, unchanged: one worker can hold several sessions, and each is one upgrade and one unlink. Nothing has to settle it when a worker dies, unlike the detached count: it lives on the port, not on the application, so it dies with the port, and the unlink that follows finds ->pair[1] already -1 and reaches no idle transition. Co-Authored-By: Claude Opus 5 --- src/nxt_port.h | 10 ++++++---- src/nxt_router.c | 42 ++++++++++++++++++++++++++++++---------- src/nxt_router_request.h | 7 +++++++ 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/nxt_port.h b/src/nxt_port.h index 1acd514fd..836598b01 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -334,17 +334,19 @@ struct nxt_port_s { /* Maximum interleave of message parts. */ uint32_t max_share; + /* + * Websocket sessions upgraded from a request this worker answered. A + * session is counted by NXT_APR_UPGRADE and uncounted by + * NXT_APR_WEBSOCKET_CLOSE, both in nxt_router_app_port_release(). + */ uint32_t active_websockets; /* * The application answered a request on this port and kept running. * Treated exactly like active_websockets by the idle transition in - * nxt_router_app_port_release(): the port stays in app->ports and in + * nxt_router_app_port_idle(): the port stays in app->ports and in * app->processes, and stays out of the idle queues, so the reaper * never sees it and it keeps counting against "processes": {"max"}. - * - * Unlike active_websockets this one is cleared again, when the - * application reports the work finished. */ uint8_t detached; diff --git a/src/nxt_router.c b/src/nxt_router.c index 8bed7bb9b..38c9e21ac 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5578,7 +5578,14 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_thread_mutex_unlock(&app->mutex); nxt_router_app_port_release(task, app, app_port, NXT_APR_UPGRADE); - req_rpc_data->apr_action = NXT_APR_CLOSE; + + /* + * The count above is undone by this action, and only by it: + * nxt_request_rpc_data_unlink() is the one path that runs for a + * websocket that ends, whichever side ends it, and ->app_port is + * still this port when it does. + */ + req_rpc_data->apr_action = NXT_APR_WEBSOCKET_CLOSE; nxt_debug(task, "stream #%uD upgrade", req_rpc_data->stream); @@ -6843,7 +6850,7 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_apr_action_t action) { int inc_use; - uint32_t got_response, dec_requests; + uint32_t got_response, dec_requests, dec_websockets; nxt_bool_t adjust_idle_timer; nxt_port_t *main_app_port; @@ -6852,6 +6859,7 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, inc_use = 0; got_response = 0; dec_requests = 0; + dec_websockets = 0; switch (action) { case NXT_APR_NEW_PORT: @@ -6870,11 +6878,21 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, case NXT_APR_CLOSE: inc_use = -1; break; + case NXT_APR_WEBSOCKET_CLOSE: + dec_websockets = 1; + inc_use = -1; + break; } - nxt_debug(task, "app '%V' release port %PI:%d: %d %d", &app->name, + nxt_debug(task, "app '%V' release port %PI:%d: %d %d %d", &app->name, port->pid, port->id, - (int) inc_use, (int) got_response); + (int) inc_use, (int) got_response, (int) dec_websockets); + + /* + * A websocket is upgraded from a request a worker has answered, so the + * port it is released on is that worker's own port, never the shared one. + */ + nxt_assert(dec_websockets == 0 || port->id != NXT_SHARED_PORT_ID); if (port->id == NXT_SHARED_PORT_ID) { nxt_thread_mutex_lock(&app->mutex); @@ -6891,6 +6909,7 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, nxt_thread_mutex_lock(&app->mutex); main_app_port->active_requests -= got_response + dec_requests; + main_app_port->active_websockets -= dec_websockets; app->active_requests -= got_response + dec_requests; if (main_app_port->pair[1] != -1 && main_app_port->app_link.next == NULL) { @@ -7910,12 +7929,15 @@ nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) * or closes. See nxt_router_app_abandon(). * * An upgraded stream is not such a request. Its accounting was already - * given back by NXT_APR_UPGRADE, so there is no slot being held and - * nothing to keep out of the idle economy; the mark would only report the - * worker "detached" for as long as the connection lives. The websocket - * state is the upgrade itself -- it is assigned beside that release, and - * nowhere else -- so a handshake still in flight or one that failed, both - * of which do hold their accounting, still reach the abandon. + * given back by NXT_APR_UPGRADE, so no worker is running it and there is + * no slot to hold; what does keep the port out of the idle economy is the + * session count, and that one is settled by the unlink below, because the + * upgrade left NXT_APR_WEBSOCKET_CLOSE as the action. The mark would + * instead report the worker "detached" for as long as the connection + * lives. The websocket state is the upgrade itself -- it is assigned + * beside that release, and nowhere else -- so a handshake still in flight + * or one that failed, both of which do hold their accounting, still reach + * the abandon. */ if (req_rpc_data->app_port != NULL && req_rpc_data->app_port->id != NXT_SHARED_PORT_ID diff --git a/src/nxt_router_request.h b/src/nxt_router_request.h index a36ef0e24..993f3c433 100644 --- a/src/nxt_router_request.h +++ b/src/nxt_router_request.h @@ -37,6 +37,13 @@ typedef enum { NXT_APR_GOT_RESPONSE, NXT_APR_UPGRADE, NXT_APR_CLOSE, + /* + * The close of a request that NXT_APR_UPGRADE turned into a websocket. + * It is the counterpart of that action: it drops the reference the way + * NXT_APR_CLOSE does and also takes the session off the worker's + * active_websockets count. + */ + NXT_APR_WEBSOCKET_CLOSE, } nxt_apr_action_t; From 3f992cd76b524941f14ccc50ac3139fc8f57f5f0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 18 Sep 2026 12:29:43 +0200 Subject: [PATCH 222/391] tests: cover the websocket release accounting Drives nxt_router_app_port_release() over the actions a websocket session passes through, on a fixture of one application and one worker port that is its own main port, and reads the session count, the idle queues and the port reference after each one. The table holds two sessions on the same worker, so it fails on a decrement that is really a clear, and a close of a plain request, so it fails on a decrement that every close does. The port rejoins the idle economy only on the last step, which is the behaviour the fix is for. The last session closes on a worker that is also ->detached, the mark 1.36.2 added for a worker the deadline gave up on. Both are conditions of the one guard in nxt_router_app_port_idle(), so the step holds the port out until the mark clears as well, and it fails if either condition is dropped from that guard. The release is static, so it is reached through a test-only wrapper, the way the detached edge handler already is. nxt_router.h now includes nxt_router_request.h for the action type that wrapper takes. Co-Authored-By: Claude Opus 5 --- auto/sources | 1 + src/nxt_router.c | 14 ++ src/nxt_router.h | 5 + src/test/nxt_router_websocket_test.c | 250 +++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 6 files changed, 275 insertions(+) create mode 100644 src/test/nxt_router_websocket_test.c diff --git a/auto/sources b/auto/sources index ea4b893ab..43eb6bbd9 100644 --- a/auto/sources +++ b/auto/sources @@ -180,6 +180,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_app_timeout_test.c \ src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_router_detached_test.c \ + src/test/nxt_router_websocket_test.c \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ src/test/nxt_proto_creating_wedge_test.c \ diff --git a/src/nxt_router.c b/src/nxt_router.c index 38c9e21ac..34ae7309f 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -6944,6 +6944,20 @@ nxt_router_app_port_release(nxt_task_t *task, nxt_app_t *app, nxt_port_t *port, } +#if (NXT_TESTS) + +/* For src/test/nxt_router_websocket_test.c. */ + +void +nxt_router_test_app_port_release(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port, nxt_apr_action_t action) +{ + nxt_router_app_port_release(task, app, port, action); +} + +#endif + + void nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) { diff --git a/src/nxt_router.h b/src/nxt_router.h index 90ebb568c..b7923dffa 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -14,6 +14,7 @@ typedef struct nxt_http_request_s nxt_http_request_t; #include +#include typedef struct nxt_http_action_s nxt_http_action_t; @@ -365,6 +366,10 @@ void nxt_router_access_log_reopen_handler(nxt_task_t *task, /* The detached edge handler, for the sender check test. */ void nxt_router_test_detached_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); + +/* The port release, for the websocket accounting test. */ +void nxt_router_test_app_port_release(nxt_task_t *task, nxt_app_t *app, + nxt_port_t *port, nxt_apr_action_t action); #endif #if (NXT_TESTS) diff --git a/src/test/nxt_router_websocket_test.c b/src/test/nxt_router_websocket_test.c new file mode 100644 index 000000000..4c350158a --- /dev/null +++ b/src/test/nxt_router_websocket_test.c @@ -0,0 +1,250 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Regression test for the websocket accounting in + * nxt_router_app_port_release() (src/nxt_router.c). + * + * A websocket upgrade counts the session on the worker's main port, and + * nxt_router_app_port_idle() keeps a port with a session out of the idle + * queues. Nothing uncounted it, so a worker that had served one upgrade + * never rejoined the idle economy: the reaper never saw it and its slot + * counted against "processes": {"max"} for the life of the process. The + * upgrade now leaves NXT_APR_WEBSOCKET_CLOSE behind for the unlink, and + * that action uncounts the session. + * + * The fixture is one application with one worker port that is its own + * ->main_app_port. ->spare_processes is 1 and ->idle_processes starts at + * 0, so the idle transition inserts into spare_ports and never arms the + * reaper timer, which keeps the release confined to the queues and the + * counters this test reads. ->pair[1] is not -1, so the transition is + * reached at all. + * + * The table walks the life of a session: the release that follows the + * upgrade, a second session on the same worker, and the two closes. A + * close of a plain request is in it as well, to show that the decrement + * belongs to the new action and not to every close. + * + * The last session closes on a worker the router has also marked + * ->detached, because both are conditions of the one guard in + * nxt_router_app_port_idle(): uncounting the session must not hand back a + * port the detached state still holds, and clearing that state must. The + * step sets the flag directly, since app->detached_processes is settled by + * the edges that own it and is not read here. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +typedef struct { + const char *name; + nxt_bool_t upgrade; + uint8_t detached; + nxt_apr_action_t action; + uint32_t active_websockets; + nxt_bool_t idle; + nxt_atomic_t use_count; +} nxt_router_websocket_test_step_t; + + +static const nxt_router_websocket_test_step_t + nxt_router_websocket_test_steps[] = +{ + { "the first upgrade", 1, 0, NXT_APR_UPGRADE, 1, 0, 5 }, + { "a second session on the same worker", 1, 0, NXT_APR_UPGRADE, 2, 0, 5 }, + { "the first session closes", 0, 0, NXT_APR_WEBSOCKET_CLOSE, 1, 0, 4 }, + { "a plain request closes", 0, 0, NXT_APR_CLOSE, 1, 0, 3 }, + { "the last session closes, detached", 0, 1, NXT_APR_WEBSOCKET_CLOSE, + 0, 0, 2 }, + { "the detached mark clears", 0, 0, NXT_APR_CLOSE, 0, 1, 1 }, +}; + + +static nxt_int_t +nxt_router_websocket_test_step(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_t *port, nxt_app_t *app, + const nxt_router_websocket_test_step_t *step) +{ + nxt_bool_t idle; + + if (step->upgrade) { + /* + * What the two handlers ahead of the release do: the acknowledgement + * counts the request on the port, see + * nxt_router_req_headers_ack_handler(), and the response counts the + * session, see nxt_router_response_ready_handler(). NXT_APR_UPGRADE + * gives the request back and leaves the session behind. + */ + nxt_thread_mutex_lock(&app->mutex); + + port->active_requests++; + port->active_websockets++; + + nxt_thread_mutex_unlock(&app->mutex); + } + + nxt_thread_mutex_lock(&app->mutex); + + port->detached = step->detached; + + nxt_thread_mutex_unlock(&app->mutex); + + nxt_router_test_app_port_release(task, app, port, step->action); + + idle = (port->idle_link.next != NULL); + + if (port->active_requests != 0 + || port->active_websockets != step->active_websockets + || idle != step->idle + || app->idle_processes != (step->idle ? 1U : 0U) + || port->use_count != step->use_count) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router websocket test: %s: websockets %uD, idle %d, " + "idle processes %uD, use count %d; expected " + "%uD, %d, %uD, %d", + step->name, port->active_websockets, (int) idle, + app->idle_processes, (int) port->use_count, + step->active_websockets, (int) step->idle, + step->idle ? 1U : 0U, (int) step->use_count); + return NXT_ERROR; + } + + return NXT_OK; +} + + +nxt_int_t +nxt_router_websocket_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_app_t *app; + nxt_uint_t i; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_bool_t app_mutex; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router websocket test started"); + + ret = NXT_ERROR; + port = NULL; + app_mutex = 0; + + task = thr->task; + task->thread = thr; + + saved_engine = thr->engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + app = nxt_mp_zalloc(mp, sizeof(nxt_app_t)); + if (nxt_slow_path(app == NULL)) { + goto done; + } + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + + thr->engine = &engine; + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "websocket-test"); + + /* Never reached by the releases below, so nothing frees the application. */ + app->use_count = 8; + + /* One spare, so the idle transition never arms the reaper timer. */ + app->spare_processes = 1; + + port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_APP); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + port->app = app; + port->main_app_port = port; + + /* + * The idle transition and the app->ports insertion both read ->pair[1], + * and nxt_port_close() must not act on the value, so set it after + * nxt_port_new() and clear it again before the port is released. + */ + port->pair[1] = 0; + + /* + * One reference for the fixture and one for each of the four releases + * below that drops one, so the port outlives the table. The first + * release takes one back when it puts the port into app->ports, which + * is why the table starts at 5. + */ + port->use_count = 4; + + for (i = 0; i < nxt_nitems(nxt_router_websocket_test_steps); i++) { + if (nxt_slow_path(nxt_router_websocket_test_step(thr, task, port, app, + &nxt_router_websocket_test_steps[i]) != NXT_OK)) + { + goto done; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router websocket test passed"); + + ret = NXT_OK; + +done: + + if (port != NULL) { + port->pair[1] = -1; + + if (port->app_link.next != NULL) { + nxt_queue_remove(&port->app_link); + port->app_link.next = NULL; + } + + if (port->idle_link.next != NULL) { + nxt_queue_remove(&port->idle_link); + port->idle_link.next = NULL; + } + + port->app = NULL; + port->main_app_port = NULL; + + port->use_count = 1; + + nxt_port_use(task, port, -1); + } + + thr->engine = saved_engine; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 59cfe443f..b2411af70 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -242,6 +242,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_websocket_test(thr) != NXT_OK) { + return 1; + } + if (nxt_main_start_process_reply_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index bc30cf439..8c596cd72 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -82,6 +82,7 @@ nxt_int_t nxt_router_start_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_app_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_detached_test(nxt_thread_t *thr); +nxt_int_t nxt_router_websocket_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); From 1554ef35141d87a142c7c34a38e7aa9261cb9fa5 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 18 Sep 2026 12:35:23 +0200 Subject: [PATCH 223/391] tests: show a websocket worker rejoining the idle economy Upgrades a session, checks over /status that the worker is out of the idle queues while it is open, closes it and checks that the worker is back. Before the fix the second check never came true: the application kept one running process and no idle one for the life of the worker. The counts are read by name rather than compared whole, because "processes" grows keys. Co-Authored-By: Claude Opus 5 --- test/test_node_websockets.py | 51 ++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index b4ebd5245..dde3452af 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -10,6 +10,7 @@ prerequisites = {'modules': {'node': 'any'}} client = ApplicationNode() +control = Control() ws = ApplicationWebsocket() @@ -1462,8 +1463,7 @@ def test_node_websockets_timeout_does_not_detach(): time.sleep(3) processes = ( - Control() - .conf_get('/status') + control.conf_get('/status') .get('applications', {}) .get('websockets/mirror', {}) .get('processes', {}) @@ -1474,3 +1474,50 @@ def test_node_websockets_timeout_does_not_detach(): ) sock.close() + + +def wait_idle(name, idle, timeout=10): + """The idle count of an application, once it settles. + + Read by name rather than compared whole: /status grows keys, and this + is about one of them. + """ + + for _ in range(timeout * 10): + processes = control.conf_get('/status')['applications'][name][ + 'processes' + ] + + if processes['idle'] == idle: + break + + time.sleep(0.1) + + return processes + + +def test_node_websockets_worker_idle_after_close(): + client.load('websockets/mirror') + + name = 'websockets/mirror' + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, 'blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'blah') + + # The worker is out of the idle economy while the session is open. + + processes = wait_idle(name, 0) + + assert processes['running'] == 1, 'running while open' + assert processes['idle'] == 0, 'busy while open' + + close_connection(sock) + + # And rejoins it when the session ends. + + processes = wait_idle(name, 1) + + assert processes['running'] == 1, 'running after close' + assert processes['idle'] == 1, 'idle after close' From bc28f594365738dc0a00598c50eaefc102d9b6f5 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 18 Sep 2026 12:36:14 +0200 Subject: [PATCH 224/391] changelog: a websocket worker is idle again when the session ends Co-Authored-By: Claude Opus 5 --- CHANGES | 9 +++++++++ docs/changes.xml | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGES b/CHANGES index 39edeb757..f27e22bd0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: an application worker that served a WebSocket upgrade never + became idle again. The router counts a session on the worker and + keeps a worker with a session out of the idle queues, but nothing + uncounted the session when it ended, so the process stayed out of + them for good: its idle timer never ran and its slot counted against + "processes": {"max"} until it died, which an application that serves + WebSockets reaches one upgrade at a time. A session is now uncounted + when it ends, whichever side ends it; the bug had appeared in 1.19.0. + *) Bugfix: an application that answered a request and kept running was counted idle. PHP does this whenever a script calls fastcgi_finish_request(): the response goes out, the script runs on, diff --git a/docs/changes.xml b/docs/changes.xml index e04e79892..de2b497d6 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,18 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +an application worker that served a WebSocket upgrade never became idle again. +The router counts a session on the worker and keeps a worker with a session out +of the idle queues, but nothing uncounted the session when it ended, so the +process stayed out of them for good: its idle timer never ran and its slot +counted against "processes": {"max"} until it died, which an application that +serves WebSockets reaches one upgrade at a time. A session is now uncounted +when it ends, whichever side ends it; the bug had appeared in 1.19.0. + + + an application that answered a request and kept running was counted idle. PHP From 6be793bd7463193496bc211bf15cee1a6b525803 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 22:53:35 +0000 Subject: [PATCH 225/391] build(deps): Bump actions/download-artifact from 4 to 8 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4 to 8. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4...v8) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '8' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c5c333073..73a5f1130 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -426,7 +426,7 @@ jobs: # flags and the same objects, compiled once for the whole run instead # of once per leg. Only this leg's own module is built below. - name: Download the core build - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: unit-core-build path: /tmp From 3bc72e19555b5142c317e8253fb15ca5d7609a66 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 16 Sep 2026 20:40:48 +0200 Subject: [PATCH 226/391] contrib: bump wasmtime 43.0.1 to 47.0.4 43.0.1 is affected by six distinct advisories: wasmtime RUSTSEC-2026-0114 (CVE-2026-44216, table allocation panic), -0222 (cross-engine type indices) and -0269 (filesystem sandbox escape via trailing slashes), plus wasmtime-wasi RUSTSEC-2026-0149 (path_open TRUNCATE bypasses FilePerms::WRITE), -0182 (fd_renumber leak) and -0188 (hard links and renames bypass destination FilePerms). 47.0.4 returns none for both crates. RUSTSEC-2026-0269 is reachable through this module: wasmtime_linker_define_wasi() defines the WASI p1 path surface and every access.filesystem entry is preopened read+write with the host path used verbatim as the guest path. The c-api build turns WASMTIME_FEATURE_WASI_HTTP off; the wasm module only uses p1 WASI and v47 defaults wasi-http on, which pulls the rustls stack into the linked library. Source compatibility checked against the v47 headers: every symbol the module calls still exists; the only signature changes are added const, and the removed declarations are the unused externref/anyref GC helpers. The tarball comes from the mirror: freeunitorg/packages.freeunitorg.github.io#10. EOL.md and pkg/eol.json now name both pins at 47.0.4. --- EOL.md | 2 +- pkg/contrib/src/wasmtime/Makefile | 1 + pkg/contrib/src/wasmtime/SHA512SUMS | 2 +- pkg/contrib/src/wasmtime/version | 2 +- pkg/eol.json | 4 ++-- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/EOL.md b/EOL.md index 85a504430..622d88a49 100644 --- a/EOL.md +++ b/EOL.md @@ -92,7 +92,7 @@ needs an upgrade or replacement decision instead. | Eclipse Jetty | `jetty-util` / `jetty-server` / `jetty-http` jars bundled by the Java module | 9.4.58.v20250814 | **Aug 2025 (EOL)** † | Jetty 9.4 lost community support on 2025-08-14 and the bundled build is its last release. Needs an upgrade-or-replace decision — Jetty 10/11 are EOL too; 12.x is the supported line. | | Eclipse ECJ (JDT batch compiler) | JSP compilation jar bundled by the Java module | 3.26.0 | none published | Tracks Eclipse releases; endoflife.date has no product for it. Pinned build is from Jun 2021; current is 3.42.0 (Jun 2025). | | ClassGraph | classpath scanning jar bundled by the Java module | latest | — | No upstream EOL schedule; pinned and bumped as needed. | -| Wasmtime (C API) | WebAssembly runtime built by `pkg/contrib` and linked by the `wasm` module | 43.0.1 | none published | Fetched as a source tarball (`pkg/contrib/src/wasmtime/version`), so `cargo audit` never sees it. Four majors behind the Rust pin below, and affected by RUSTSEC-2026-0269 and -0222 — see #400. | +| Wasmtime (C API) | WebAssembly runtime built by `pkg/contrib` and linked by the `wasm` module | 47.0.4 | none published | Fetched as a source tarball (`pkg/contrib/src/wasmtime/version`), so `cargo audit` never sees it; the `Audit (cargo)` pins job queries OSV for the pinned version instead. Same 47.0.4 as the Rust pin below; the 43.0.1 bump closed RUSTSEC-2026-0269, -0222, -0114 and three wasmtime-wasi path/permission advisories — see #400. | | Wasmtime (Rust crate) | WebAssembly runtime used by the `wasm-wasi-component` module | 47.0.4 | none published | `src/wasm-wasi-component/Cargo.lock`, audited by the `Audit (cargo)` workflow on every lock. Bumped with the advisories it closes. | ## Rules diff --git a/pkg/contrib/src/wasmtime/Makefile b/pkg/contrib/src/wasmtime/Makefile index dce9ad3b8..996cfd4ab 100644 --- a/pkg/contrib/src/wasmtime/Makefile +++ b/pkg/contrib/src/wasmtime/Makefile @@ -23,6 +23,7 @@ wasmtime: wasmtime-v$(WASMTIME_VERSION)-src.tar.gz .sum-wasmtime .wasmtime: wasmtime cd $< && cmake \ -DCMAKE_INSTALL_LIBDIR=lib \ + -DWASMTIME_FEATURE_WASI_HTTP=OFF \ -S crates/c-api \ -B target/c-api \ --install-prefix "$(TOPSRC)/wasmtime/artifacts" diff --git a/pkg/contrib/src/wasmtime/SHA512SUMS b/pkg/contrib/src/wasmtime/SHA512SUMS index 0a0d35eed..4a150cf06 100644 --- a/pkg/contrib/src/wasmtime/SHA512SUMS +++ b/pkg/contrib/src/wasmtime/SHA512SUMS @@ -1 +1 @@ -1a7d3eba2e39ecb148771908714a799ec8511b9fff63c5e8ab30000ddb6a5380a575b9d479b4fc72eb5aa52bed78ee6de82df7437bf2c7e0c357ab2cd2aebef0 wasmtime-v43.0.1-src.tar.gz \ No newline at end of file +3a27dada5ed2263f4ce769cd81dc8d7d5cb79ec3ccf02bd745a8231302dae04e5f6d58f38d4c0a3201dd4f9022efcf247548f62d0be7baedcb128af29b40b078 wasmtime-v47.0.4-src.tar.gz \ No newline at end of file diff --git a/pkg/contrib/src/wasmtime/version b/pkg/contrib/src/wasmtime/version index 382960944..dd0483ab4 100644 --- a/pkg/contrib/src/wasmtime/version +++ b/pkg/contrib/src/wasmtime/version @@ -1 +1 @@ -WASMTIME_VERSION := 43.0.1 +WASMTIME_VERSION := 47.0.4 diff --git a/pkg/eol.json b/pkg/eol.json index 62ef44d3b..3affc4aff 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -129,9 +129,9 @@ }, "wasmtime_c_api": { "role": "WebAssembly runtime built by pkg/contrib and linked by the wasm module (pkg/contrib/src/wasmtime/version, WASMTIME_VERSION)", - "version": "43.0.1", + "version": "47.0.4", "eol": null, - "eol_note": "Fetched as a source tarball from packages.freeunit.org, so cargo audit never sees it: it reads Cargo.lock files only. Four majors behind the wasmtime_crate pin, and affected by RUSTSEC-2026-0269 (filesystem sandbox escape) and RUSTSEC-2026-0222 (stores mix up type indices) -- both list per-branch patches that skip 43.x and declare unaffected = []. Tracked in freeunitorg/freeunit#400.", + "eol_note": "Fetched as a source tarball from packages.freeunit.org, so cargo audit never sees it: it reads Cargo.lock files only. Covered by the Audit (cargo) pins job, which queries OSV for the version in pkg/contrib/src/wasmtime/version, so an advisory here arrives as a red job rather than silence. Same 47.0.4 as the wasmtime_crate pin; the 43.0.1 bump closed RUSTSEC-2026-0269 (filesystem sandbox escape), -0222, -0114, and wasmtime-wasi -0149, -0182 and -0188 (WASI path and file-permission bypasses). Tracked in freeunitorg/freeunit#400.", "source": "https://github.com/bytecodealliance/wasmtime/releases" }, "wasmtime_crate": { From af310710b28f67df629116e16215456c19fe69b7 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 16 Sep 2026 20:40:52 +0200 Subject: [PATCH 227/391] ci(audit): check tarball-pinned dependencies against OSV cargo audit reads Cargo.lock files only, so the wasmtime source tarball that pkg/contrib fetches has never had an automated advisory check. check-advisories.py reads WASMTIME_VERSION, queries OSV for wasmtime and wasmtime-wasi at that version, and refuses to trust the pin unless pkg/eol.json and SHA512SUMS agree with the version file. Run against 43.0.1 it reports the six advisories the bump closes, including three wasmtime-wasi path/permission issues that #400 did not name; 47.0.4 is clean. The new pins job joins the existing weekly schedule, and the path filters cover pkg/contrib/src/*/version and SHA512SUMS so a pin bump cannot skip it. --- .github/workflows/audit.yml | 29 +++++++-- pkg/contrib/check-advisories.py | 108 ++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 pkg/contrib/check-advisories.py diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index b21dd7c98..a059cd858 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -1,15 +1,20 @@ name: "Audit (cargo)" -# Every tracked Cargo.lock is checked against the RustSec advisory database. -# The weekly trigger is what makes this a gate rather than a formality: a new -# advisory can land against a lock file nobody has touched, and without the -# schedule the audit would only run when a lock changes. +# Every tracked Cargo.lock is checked against the RustSec advisory database, +# and every pkg/contrib source-tarball pin is checked against OSV. The weekly +# trigger is what makes this a gate rather than a formality: a new advisory can +# land against a file nobody has touched, and without the schedule the audit +# would only run when a lock or a pin changes. on: pull_request: paths: - '**/Cargo.lock' - '**/Cargo.toml' + - 'pkg/contrib/src/*/version' + - 'pkg/contrib/src/*/SHA512SUMS' + - 'pkg/contrib/check-advisories.py' + - 'pkg/eol.json' # A change to this file must run this file. - .github/workflows/audit.yml push: @@ -18,6 +23,10 @@ on: paths: - '**/Cargo.lock' - '**/Cargo.toml' + - 'pkg/contrib/src/*/version' + - 'pkg/contrib/src/*/SHA512SUMS' + - 'pkg/contrib/check-advisories.py' + - 'pkg/eol.json' - .github/workflows/audit.yml schedule: - cron: "0 6 * * 1" @@ -80,3 +89,15 @@ jobs: # open. - name: Audit ${{ matrix.lock }} run: cargo audit --deny warnings --file '${{ matrix.lock }}' + + pins: + # A dependency pinned as a source tarball has no Cargo.lock, so cargo + # audit cannot see it; this checks those pins against OSV, which mirrors + # the RustSec database and carries the GHSA/CVE aliases. + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + + - name: Check source-tarball pins + run: python3 pkg/contrib/check-advisories.py diff --git a/pkg/contrib/check-advisories.py b/pkg/contrib/check-advisories.py new file mode 100644 index 000000000..bc3eca6b8 --- /dev/null +++ b/pkg/contrib/check-advisories.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Check tarball-pinned dependencies against OSV advisories. + +`cargo audit` reads Cargo.lock files only, so a dependency pinned as a source +tarball under pkg/contrib is invisible to it. This script reads the version +pin and asks OSV, which mirrors the RustSec advisory database and also carries +the GHSA/CVE aliases. + +It also checks that the three places naming the pin agree: + - pkg/contrib/src/wasmtime/version + - pkg/contrib/src/wasmtime/SHA512SUMS + - pkg/eol.json (dependencies.wasmtime_c_api) + +Exit codes: 0 clean, 1 advisories found, 2 configuration or query failure. +""" +import json +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +VERSION_FILE = REPO / "pkg/contrib/src/wasmtime/version" +SUMS_FILE = REPO / "pkg/contrib/src/wasmtime/SHA512SUMS" +EOL_FILE = REPO / "pkg/eol.json" +OSV_URL = "https://api.osv.dev/v1/query" + +# The contrib tarball is the wasmtime workspace; pinning it pins both crates +# that the C API links. +CRATES = ("wasmtime", "wasmtime-wasi") + + +def read_version() -> str: + match = re.search(r"WASMTIME_VERSION\s*:=\s*(\S+)", VERSION_FILE.read_text()) + if not match: + fail(f"no WASMTIME_VERSION in {VERSION_FILE.relative_to(REPO)}") + return match.group(1) + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + sys.exit(2) + + +def check_sums(version: str) -> None: + tarball = f"wasmtime-v{version}-src.tar.gz" + pattern = re.compile(rf"\s{tarball}$", re.MULTILINE) + if not pattern.search(SUMS_FILE.read_text()): + fail(f"{SUMS_FILE.relative_to(REPO)} has no digest for {tarball}") + + +def check_eol_json(version: str) -> None: + config = json.loads(EOL_FILE.read_text()) + pinned = config["dependencies"]["wasmtime_c_api"]["version"] + if pinned != version: + fail( + f"{EOL_FILE.relative_to(REPO)} says wasmtime_c_api {pinned}, but the " + f"pin is {version}" + ) + + +def query(crate: str, version: str) -> list: + body = json.dumps( + {"package": {"name": crate, "ecosystem": "crates.io"}, "version": version} + ).encode() + request = urllib.request.Request( + OSV_URL, data=body, headers={"Content-Type": "application/json"} + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response).get("vulns", []) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + fail(f"OSV query for {crate} {version} failed: {error}") + + +def main() -> int: + version = read_version() + check_sums(version) + check_eol_json(version) + + found = False + seen = set() + for crate in CRATES: + for vuln in query(crate, version): + aliases = vuln.get("aliases", []) + # OSV carries the GHSA and RUSTSEC records of one advisory as two + # entries that alias each other; report each advisory once. + key = tuple(sorted({vuln["id"], *aliases})) + if key in seen: + continue + seen.add(key) + found = True + alias_text = ", ".join(aliases) + suffix = f" ({alias_text})" if alias_text else "" + print(f"{crate} {version}: {vuln['id']}{suffix}") + if vuln.get("summary"): + print(f" {vuln['summary']}") + + if found: + return 1 + + print(f"{', '.join(CRATES)} {version}: no advisories") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 179786574b04567903806ca72b4756866a7215e2 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 16 Sep 2026 20:40:52 +0200 Subject: [PATCH 228/391] test(wasm): run a C wasm guest, and cover access.filesystem The legacy wasm module had compile-only CI and no runtime test at all. test/test_wasm.py builds test/wasm/hello/hello.c at test time against pkg/contrib's wasi-sysroot and libunit-wasm source, then drives it through unitd: a fixed body, a file read inside an access.filesystem preopen, and a denied read outside it including a symlink with the trailing slash RUSTSEC-2026-0269 abused. The app config names the module init and end handlers as well as request, malloc and free. nxt_wasm_setup() (src/wasm/nxt_wasm.c:408) wires NXT_WASM_FH_MODULE_INIT/_END only from the config, and NXT_WASM_DO_HOOK is a no-op on a NULL func_name, so without them the guest's init hook never runs and request_buf is still NULL at hello.c:117 when luw_set_req_buf() reaches its memcpy() -- which does not allocate under LUW_SRB_NONE. That copy would land at address 0 of the linear memory, ordinary writable memory that --stack-first leaves at the untouched deep end of an 8 MB shadow stack, so the tests would pass anyway. The guest link flags are built as one explicit -Wl, argument: clang splits that on commas, so "-z stack-size=8388608" reaches the linker as a linker option instead of an unknown clang argument. The wasm build-test leg now installs pytest and the guest toolchain, configures with --rpath so pytest can dlopen the module, runs unprivileged so the guest build is runner-owned, and fails if no test_wasm.py test passed instead of going green on a skip. Its timeout grows to 45 minutes because the first run on a bumped pin rebuilds wasmtime from source. The compiler-rt package name takes the major component of clang -dumpversion, because on Ubuntu that prints the full release (21.1.8 here, 20.1.8 in the wasi-sdk) and libclang-rt-21.1.8-dev-wasm32 does not exist. Local: pytest test/test_wasm.py -> 3 passed against 47.0.4. Co-Authored-By: Claude Opus 5 --- .github/workflows/build-test.yml | 64 +++++++++++-- test/test_app_lifecycle.py | 8 +- test/test_wasm.py | 63 +++++++++++++ test/unit/applications/lang/wasm.py | 102 ++++++++++++++++++++ test/unit/check/clang_wasm.py | 5 + test/unit/check/discover_available.py | 2 + test/wasm/hello/hello.c | 128 ++++++++++++++++++++++++++ 7 files changed, 359 insertions(+), 13 deletions(-) create mode 100644 test/test_wasm.py create mode 100644 test/unit/applications/lang/wasm.py create mode 100644 test/unit/check/clang_wasm.py create mode 100644 test/wasm/hello/hello.c diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 73a5f1130..e15e338ca 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -206,7 +206,9 @@ jobs: test: needs: [prepare, core] runs-on: ${{ matrix.os }} - timeout-minutes: 30 + # 45, not 30: the wasm leg now runs the suite on a cold contrib wasmtime + # build (the cache key moves with the pin) and builds the guest toolchain. + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -629,7 +631,13 @@ jobs: uses: actions/cache@v6 with: path: pkg/contrib/wasmtime/artifacts - key: wasmtime-${{ steps.metadata.outputs.wasmtime_version }}-${{ runner.os }}-${{ runner.arch }} + # Every file that decides what gets downloaded, verified and + # built, not just the version. The Makefile matters most: its cmake + # flags decide which features the library carries, so without it a + # feature change with no version bump would restore the old library + # and skip the rebuild, leaving the tests and the audit gate + # disagreeing about what was linked. + key: wasmtime-${{ steps.metadata.outputs.wasmtime_version }}-${{ hashFiles('pkg/contrib/src/wasmtime/*') }}-${{ runner.os }}-${{ runner.arch }} - name: Make wasmtime run: | @@ -638,7 +646,7 @@ jobs: - name: Configure wasm run: | - ./configure wasm --include-path=pkg/contrib/wasmtime/artifacts/include --lib-path=pkg/contrib/wasmtime/artifacts/lib + ./configure wasm --rpath --include-path=pkg/contrib/wasmtime/artifacts/include --lib-path=pkg/contrib/wasmtime/artifacts/lib if: steps.metadata.outputs.module == 'wasm' - name: Make wasm @@ -646,6 +654,32 @@ jobs: make wasm if: steps.metadata.outputs.module == 'wasm' + # The guest fixture is compiled at test time from pkg/contrib sources: + # wasi-sysroot for wasi-libc and libunit-wasm for the request ABI. + - name: Cache wasi-sysroot + id: cache-wasi-sysroot + if: steps.metadata.outputs.module == 'wasm' + uses: actions/cache@v6 + with: + path: | + pkg/contrib/wasi-sysroot + pkg/contrib/libunit-wasm + # Every file that decides what gets downloaded, verified and built, + # not just the version: otherwise a checksum or build-rule change + # restores the old tree and skips the make below. + key: wasi-sysroot-${{ hashFiles('pkg/contrib/src/wasi-sysroot/*', 'pkg/contrib/src/libunit-wasm/*') }}-${{ runner.os }}-${{ runner.arch }} + + - name: Make wasi-sysroot + run: | + make -C pkg/contrib .wasi-sysroot libunit-wasm + if: steps.metadata.outputs.module == 'wasm' && steps.cache-wasi-sysroot.outputs.cache-hit != 'true' + + - name: Install wasm guest toolchain + run: | + sudo apt install -y clang lld + sudo apt install -y "libclang-rt-$(clang -dumpversion | cut -d. -f1)-dev-wasm32" + if: steps.metadata.outputs.module == 'wasm' + ## ## wasm-wasi-component ## @@ -725,8 +759,6 @@ jobs: run: | sudo apt install -y python3-pytest - if: steps.metadata.outputs.module != 'wasm' - - name: Run ${{ steps.metadata.outputs.module }} tests # GitHub runners are UTC, where localtime() and gmtime() agree, so a # UTC-only run cannot tell a correct HTTP-date from one rendered in @@ -743,12 +775,28 @@ jobs: if [ "${{ steps.metadata.outputs.module }}" == "python" ]; then export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" fi - if [ "${{ matrix.build }}" == "wasm-wasi-component" ]; then - pytest-3 --print-log ${{ steps.metadata.outputs.testpath }} + # The wasm legs build their guest fixture as the runner user, so + # they run unprivileged to keep the result reachable by pytest; the + # others install applications as root. + if [ "${{ matrix.build }}" == "wasm-wasi-component" ] || [ "${{ matrix.build }}" == "wasm" ]; then + set -o pipefail + pytest-3 --print-log ${{ steps.metadata.outputs.testpath }} 2>&1 | tee "$RUNNER_TEMP/pytest.out" else sudo -E pytest-3 --print-log ${{ steps.metadata.outputs.testpath }} fi - if: steps.metadata.outputs.module != 'wasm' + + # check_prerequisites() skips the whole file when the clang/wasi-sysroot + # fixture cannot be built, so an incomplete toolchain would leave this + # leg green while it covers nothing. At least one smoke test must have + # actually passed. + - name: Fail if the wasm smoke test skipped + if: ${{ !cancelled() && steps.metadata.outputs.module == 'wasm' }} + run: | + grep -qE 'test_wasm\.py::.*PASSED' "$RUNNER_TEMP/pytest.out" || { + echo "no test_wasm.py test passed; the guest toolchain skipped it:" >&2 + grep -E 'SKIPPED|feature' "$RUNNER_TEMP/pytest.out" >&2 || true + exit 1 + } # test_capget_fallback.py needs a unitd that actually calls capget(), # which a root one never does, so every leg above skips it -- the file diff --git a/test/test_app_lifecycle.py b/test/test_app_lifecycle.py index 4944179de..a48c26277 100644 --- a/test/test_app_lifecycle.py +++ b/test/test_app_lifecycle.py @@ -110,11 +110,9 @@ def load(self, script, **kwargs): # a request that fills a segment leaves it unable to build a response at all. # That is a property of the sample app, not of the port handling under test. # -# The legacy `wasm` runtime is absent deliberately: the suite has no client -# for it (only test/unit/applications/lang/wasm_component.py exists), its -# build-test leg does not install or run pytest at all -# (`if: module != 'wasm'`), and sanitize.yml has no wasm leg. A row here -# would skip in every leg that exists. +# The legacy `wasm` runtime is absent: its client (test/test_wasm.py) is a +# WASI smoke guest, not a body-mirroring app, and sanitize.yml has no wasm +# leg. A row here would skip in every leg that exists. RUNTIMES = { 'libunit': (ApplicationLibunit, 'libunit', [], [], False, False), 'go': (ApplicationGo, 'mirror', ['go'], [], True, True), diff --git a/test/test_wasm.py b/test/test_wasm.py new file mode 100644 index 000000000..f7f7dcd86 --- /dev/null +++ b/test/test_wasm.py @@ -0,0 +1,63 @@ +from pathlib import Path + +from unit.applications.lang.wasm import ApplicationWasm +from unit.check.check_prerequisites import check_prerequisites +from unit.option import option + +prerequisites = {'modules': {'wasm': 'any'}, 'features': {'clang_wasm': True}} + +check_prerequisites(prerequisites) + +client = ApplicationWasm() + + +def test_wasm_hello(): + client.load('hello') + + resp = client.get() + + assert resp['status'] == 200 + assert resp['body'] == 'Hello from wasm\n' + + +def test_wasm_access_filesystem(): + data = Path(option.temp_dir) / 'wasm_data' + data.mkdir() + (data / 'hello.txt').write_text('from the preopen\n') + + client.load('hello', access_filesystem=[f'{data}/']) + + resp = client.get(url=f'{data}/hello.txt') + + assert resp['status'] == 200 + assert resp['body'] == 'from the preopen\n' + + +def test_wasm_access_filesystem_denied(): + data = Path(option.temp_dir) / 'wasm_data' + data.mkdir() + (data / 'inside.txt').write_text('inside the preopen\n') + outside = Path(option.temp_dir) / 'outside.txt' + outside.write_text('outside the preopen\n') + + # A symlink inside the preopen pointing outside it does not escape, with + # or without the trailing slash RUSTSEC-2026-0269 abused. + (data / 'link').symlink_to(outside) + + client.load('hello', access_filesystem=[f'{data}/']) + + # The guest answers 404 for any fopen() failure, so a broken preopen would + # pass every assertion below. Read a real file first. + resp = client.get(url=f'{data}/inside.txt') + + assert resp['status'] == 200 + assert resp['body'] == 'inside the preopen\n' + + resp = client.get(url=str(outside)) + + assert resp['status'] == 404 + + for url in (f'{data}/link', f'{data}/link/'): + resp = client.get(url=url) + + assert resp['status'] == 404, url diff --git a/test/unit/applications/lang/wasm.py b/test/unit/applications/lang/wasm.py new file mode 100644 index 000000000..9d627a7fa --- /dev/null +++ b/test/unit/applications/lang/wasm.py @@ -0,0 +1,102 @@ +from pathlib import Path +import shutil +import subprocess +from urllib.parse import quote + +from unit.applications.proto import ApplicationProto +from unit.option import option + + +class ApplicationWasm(ApplicationProto): + @staticmethod + def prepare_env(script): + clang = shutil.which('clang') + root = Path(option.current_dir) + sysroot = root / 'pkg/contrib/wasi-sysroot' + include = root / 'pkg/contrib/libunit-wasm/src/c/include' + library = root / 'pkg/contrib/libunit-wasm/src/c/libunit-wasm.c' + source = Path(option.test_dir) / 'wasm' / script / f'{script}.c' + + if ( + clang is None + or not sysroot.is_dir() + or not include.is_dir() + or not library.is_file() + or not source.is_file() + ): + return None + + temp_dir = Path(option.temp_dir) / 'wasm' + temp_dir.mkdir(parents=True, exist_ok=True) + + output = temp_dir / f'{script}.wasm' + + # One -Wl, argument: clang splits it on commas and hands each piece to + # the linker, so "-z stack-size=..." stays a linker option. + link_flags = '-Wl,' + ','.join( + [ + '--no-entry', + '--export=__heap_base', + '--export=__data_end', + '--export=malloc', + '--export=free', + '--stack-first', + '-z', + 'stack-size=8388608', + ] + ) + + command = [ + clang, + f'-I{include}', + f'--sysroot={sysroot}', + '--target=wasm32-wasi', + '-O2', + '-g', + '-std=gnu11', + '-fno-common', + link_flags, + '-mexec-model=reactor', + '--rtlib=compiler-rt', + str(source), + str(library), + '-o', + str(output), + ] + + try: + subprocess.check_output(command, stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + return output + + def load(self, script, access_filesystem=None, **kwargs): + module = self.prepare_env(script) + + app = { + 'type': 'wasm', + 'processes': {'spare': 0}, + 'module': str(module), + 'request_handler': 'luw_request_handler', + 'malloc_handler': 'luw_malloc_handler', + 'free_handler': 'luw_free_handler', + # Without these two the guest's init hook never runs, so + # request_buf stays NULL and luw_set_req_buf() memcpy()s to + # address 0 of the linear memory instead of the allocation. + 'module_init_handler': 'luw_module_init_handler', + 'module_end_handler': 'luw_module_end_handler', + } + + if access_filesystem is not None: + app['access'] = {'filesystem': access_filesystem} + + self._load_conf( + { + 'listeners': { + '*:8080': {'pass': f'applications/{quote(script, "")}'} + }, + 'applications': {script: app}, + }, + **kwargs, + ) diff --git a/test/unit/check/clang_wasm.py b/test/unit/check/clang_wasm.py new file mode 100644 index 000000000..fb8db6749 --- /dev/null +++ b/test/unit/check/clang_wasm.py @@ -0,0 +1,5 @@ +from unit.applications.lang.wasm import ApplicationWasm + + +def check_clang_wasm(): + return ApplicationWasm.prepare_env('hello') is not None diff --git a/test/unit/check/discover_available.py b/test/unit/check/discover_available.py index 99e636047..c78dde1b1 100644 --- a/test/unit/check/discover_available.py +++ b/test/unit/check/discover_available.py @@ -3,6 +3,7 @@ from unit.check.cargo_component import check_cargo_component from unit.check.chroot import check_chroot +from unit.check.clang_wasm import check_clang_wasm from unit.check.go import check_go from unit.check.isolation import check_isolation from unit.check.njs import check_njs @@ -47,5 +48,6 @@ def discover_available(unit): option.available['features']['cargo_component'] = check_cargo_component() option.available['features']['chroot'] = check_chroot() + option.available['features']['clang_wasm'] = check_clang_wasm() option.available['features']['isolation'] = check_isolation() option.available['features']['unix_abstract'] = check_unix_abstract() diff --git a/test/wasm/hello/hello.c b/test/wasm/hello/hello.c new file mode 100644 index 000000000..e30080ec0 --- /dev/null +++ b/test/wasm/hello/hello.c @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +/* + * A small libunit-wasm guest for the legacy wasm runtime's smoke tests. + * + * "/" answers with a fixed body. Any other path names a file the guest + * tries to open; the host preopens every "access.filesystem" entry with the + * same path in the guest and hard-coded read+write permissions, so a path + * inside a preopen reads and anything else fails. The relative-offset + * constants below come from the upstream luw-echo-request example. + */ + +#include +#include +#include + +#include "unit/unit-wasm.h" + +#define RESPONSE_OFFSET 4096 + +static u8 *request_buf; + +__luw_export_name("luw_module_init_handler") +void luw_module_init_handler(void) +{ + request_buf = malloc(luw_mem_get_init_size()); +} + +__luw_export_name("luw_module_end_handler") +void luw_module_end_handler(void) +{ + free(request_buf); +} + +static void +send_simple(luw_ctx_t *ctx, luw_http_status_t status, const char *body) +{ + char clen[32]; + + snprintf(clen, sizeof(clen), "%zu", strlen(body)); + + luw_http_set_response_status(status); + luw_http_init_headers(ctx, 2, 0); + luw_http_add_header(ctx, "Content-Type", "text/plain"); + luw_http_add_header(ctx, "Content-Length", clen); + luw_http_send_headers(ctx); + luw_mem_writep_data(ctx, (const u8 *)body, strlen(body)); + luw_http_send_response(ctx); + luw_http_response_end(); +} + +static void +send_file(luw_ctx_t *ctx, const char *path) +{ + char clen[32]; + u8 *data; + long size; + FILE *file; + + file = fopen(path, "rb"); + if (file == NULL) { + send_simple(ctx, LUW_HTTP_NOT_FOUND, "not found\n"); + return; + } + + if (fseek(file, 0, SEEK_END) != 0) { + fclose(file); + send_simple(ctx, LUW_HTTP_INTERNAL_SERVER_ERROR, "seek failed\n"); + return; + } + + size = ftell(file); + rewind(file); + + if (size < 0) { + fclose(file); + send_simple(ctx, LUW_HTTP_INTERNAL_SERVER_ERROR, "tell failed\n"); + return; + } + + data = malloc(size); + if (data == NULL) { + fclose(file); + send_simple(ctx, LUW_HTTP_INTERNAL_SERVER_ERROR, "out of memory\n"); + return; + } + + if (fread(data, 1, size, file) != (size_t) size) { + free(data); + fclose(file); + send_simple(ctx, LUW_HTTP_INTERNAL_SERVER_ERROR, "read failed\n"); + return; + } + + fclose(file); + + snprintf(clen, sizeof(clen), "%ld", size); + + luw_http_init_headers(ctx, 2, 0); + luw_http_add_header(ctx, "Content-Type", "application/octet-stream"); + luw_http_add_header(ctx, "Content-Length", clen); + luw_http_send_headers(ctx); + luw_mem_writep_data(ctx, data, size); + luw_http_send_response(ctx); + luw_http_response_end(); + + free(data); +} + +__luw_export_name("luw_request_handler") +int luw_request_handler(u8 *addr) +{ + luw_ctx_t ctx; + const char *path; + + luw_init_ctx(&ctx, addr, RESPONSE_OFFSET); + luw_set_req_buf(&ctx, &request_buf, LUW_SRB_NONE); + + path = luw_get_http_path(&ctx); + + if (strcmp(path, "/") == 0) { + send_simple(&ctx, LUW_HTTP_OK, "Hello from wasm\n"); + } else { + send_file(&ctx, path); + } + + return 0; +} From eb1767c971971434d39bd40f8ae6905fc36d6cf2 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 18 Sep 2026 14:12:38 +0200 Subject: [PATCH 229/391] ci: audit the pinned Wasmtime source lock The wasmtime source tarball carries its own Cargo.lock, so the pins job can audit it directly instead of only asking OSV about the pin. That lock covers the whole Wasmtime workspace, though: the CLI, the fuzzing targets and wasi-http are all in it, while pkg/contrib builds one C API library with WASI_HTTP off. A plain `cargo audit --deny warnings` over it reports seven findings, five of them against crates the library never links. The archive cannot be repaired either. It ships .cargo/config.toml with replace-with = "vendored-sources" over a vendor/ directory, so `cargo update --precise` inside it is a no-op: the vendor set holds the pinned versions and nothing else. So check-lock-audit.py splits the report. A finding against a crate the C API library links fails the job unless wasmtime-lock-audit.allow.json accepts that exact version with a reason; a finding outside the linked graph is printed and ignored. The linked graph is computed rather than declared: the script resolves the cmake feature defaults against the -D flags in pkg/contrib/src/wasmtime/Makefile and runs `cargo tree --offline --locked`, which needs no network and no build. A pin bump that starts linking an affected crate therefore turns the job red on its own, and the version in each allowlist key means an acceptance is never inherited silently. The job triggers on pkg/contrib/src/*/Makefile as well, because the -D flags there decide what counts as linked. Against 47.0.4 that leaves two entries: crossbeam-epoch 0.9.18 (RUSTSEC-2026-0204), linked through rayon for parallel-compilation, where the bug is in a fmt::Pointer impl nothing calls and which wasmtime 48.x already resolves to 0.9.20; and yanked chacha20 0.10.0, linked through rand from wasmtime-wasi, where the yank carries no advisory. h2, rustls, spin, instant and paste are in the lock and not in the library. `cargo tree` resolves every target rather than the host triple, so a crate linked only on another architecture cannot read as unlinked on an x86_64 runner; that widens the 47.0.4 graph from 216 packages to 247 and reclassifies none of the seven findings. The script also refuses to run if features.cmake stops seeding --no-default-features or CMakeLists.txt stops passing ${WASMTIME_FEATURES}, because the reconstruction is only valid while cmake drives cargo that way. The job caches the archive on its committed digest instead of fetching 190 MB on every run and every Monday, retries the download so a mirror blip and an advisory do not both show up as a red job, and pins the toolchain with rustup update stable like the sibling audit job. Its path filters name pkg/contrib/src/wasmtime explicitly: both scripts hardcode that pin, so firing the job for an njs change would have reported a green covering nothing. test_check_lock_audit.py covers the gate, including the two traps: a yanked crate carries no advisory id, so the allowlist keys on the kind and version, and a warning kind the script has never seen is still gated. It lives beside the script because test/conftest.py starts a Unit server for every session and this job never builds one. Co-Authored-By: Claude Opus 5 --- .github/workflows/audit.yml | 97 +++++- pkg/contrib/check-lock-audit.py | 293 +++++++++++++++++ pkg/contrib/test_check_lock_audit.py | 348 +++++++++++++++++++++ pkg/contrib/wasmtime-lock-audit.allow.json | 26 ++ pkg/eol.json | 2 +- 5 files changed, 757 insertions(+), 9 deletions(-) create mode 100755 pkg/contrib/check-lock-audit.py create mode 100644 pkg/contrib/test_check_lock_audit.py create mode 100644 pkg/contrib/wasmtime-lock-audit.allow.json diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index a059cd858..95f50ad50 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -11,9 +11,17 @@ on: paths: - '**/Cargo.lock' - '**/Cargo.toml' - - 'pkg/contrib/src/*/version' - - 'pkg/contrib/src/*/SHA512SUMS' + # Only wasmtime: both scripts hardcode that pin, so firing this job + # for an njs or wasi-sysroot change would report a green that covers + # nothing. The Makefile is here because its build flags decide which + # crates the audit gate treats as linked. + - 'pkg/contrib/src/wasmtime/version' + - 'pkg/contrib/src/wasmtime/SHA512SUMS' + - 'pkg/contrib/src/wasmtime/Makefile' - 'pkg/contrib/check-advisories.py' + - 'pkg/contrib/check-lock-audit.py' + - 'pkg/contrib/test_check_lock_audit.py' + - 'pkg/contrib/wasmtime-lock-audit.allow.json' - 'pkg/eol.json' # A change to this file must run this file. - .github/workflows/audit.yml @@ -23,9 +31,17 @@ on: paths: - '**/Cargo.lock' - '**/Cargo.toml' - - 'pkg/contrib/src/*/version' - - 'pkg/contrib/src/*/SHA512SUMS' + # Only wasmtime: both scripts hardcode that pin, so firing this job + # for an njs or wasi-sysroot change would report a green that covers + # nothing. The Makefile is here because its build flags decide which + # crates the audit gate treats as linked. + - 'pkg/contrib/src/wasmtime/version' + - 'pkg/contrib/src/wasmtime/SHA512SUMS' + - 'pkg/contrib/src/wasmtime/Makefile' - 'pkg/contrib/check-advisories.py' + - 'pkg/contrib/check-lock-audit.py' + - 'pkg/contrib/test_check_lock_audit.py' + - 'pkg/contrib/wasmtime-lock-audit.allow.json' - 'pkg/eol.json' - .github/workflows/audit.yml schedule: @@ -91,13 +107,78 @@ jobs: run: cargo audit --deny warnings --file '${{ matrix.lock }}' pins: - # A dependency pinned as a source tarball has no Cargo.lock, so cargo - # audit cannot see it; this checks those pins against OSV, which mirrors - # the RustSec database and carries the GHSA/CVE aliases. + # A dependency pinned as a source tarball has a lock file inside the + # archive, but it is not tracked in this repository. Check the direct + # source-package pins against OSV, then audit the archived lock. That + # lock covers the whole Wasmtime workspace while Unit links one C API + # library, so check-lock-audit.py fails only on a finding against a crate + # that library actually links, and only when the allowlist does not carry + # a reason for that exact version. runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 steps: - uses: actions/checkout@v5 + - run: rustup update stable + - name: Check source-tarball pins run: python3 pkg/contrib/check-advisories.py + + - name: Test the lock audit gate + run: | + pip install --disable-pip-version-check --quiet pytest + python3 -m pytest pkg/contrib/test_check_lock_audit.py + + - name: Install cargo-audit + uses: taiki-e/install-action@v2 + with: + tool: cargo-audit + + - name: Resolve the Wasmtime pin + id: pin + run: | + version="$(sed -n -E 's/^WASMTIME_VERSION := ([0-9.]+)$/\1/p' \ + pkg/contrib/src/wasmtime/version)" + test -n "$version" + echo "archive=wasmtime-v${version}-src.tar.gz" >> "$GITHUB_OUTPUT" + + # The archive is immutable and its digest is committed, so this is one + # download per pin instead of 190 MB on every run and every Monday. The + # digest is checked below either way, so a poisoned entry still fails. + - name: Cache the Wasmtime source archive + uses: actions/cache@v4 + with: + path: ~/wasmtime-archive + key: >- + wasmtime-src-${{ hashFiles('pkg/contrib/src/wasmtime/SHA512SUMS') }} + + - name: Unpack the Wasmtime source archive + run: | + archive="${{ steps.pin.outputs.archive }}" + mkdir -p ~/wasmtime-archive + # --retry because on a scheduled gate a mirror blip and an advisory + # must not both show up as a red job. + test -f ~/"wasmtime-archive/$archive" || \ + curl --fail --location --show-error --retry 3 --retry-delay 5 \ + --output ~/"wasmtime-archive/$archive" \ + "https://packages.freeunit.org/wasmtime/$archive" + digest="$(awk -v archive="$archive" '$2 == archive { print $1 }' \ + pkg/contrib/src/wasmtime/SHA512SUMS)" + test -n "$digest" + printf '%s %s\n' "$digest" ~/"wasmtime-archive/$archive" \ + | sha512sum --check + mkdir "$RUNNER_TEMP/wasmtime" + tar -xzf ~/"wasmtime-archive/$archive" --strip-components=1 \ + --directory "$RUNNER_TEMP/wasmtime" + + - name: Audit the Wasmtime source lock + run: | + # cargo audit exits non-zero on any finding, and the archived lock + # covers the whole Wasmtime workspace, so the report is the input to + # the gate rather than the verdict. An empty report is a failure: + # check-lock-audit.py rejects anything that is not a cargo audit run. + cargo audit --json --file "$RUNNER_TEMP/wasmtime/Cargo.lock" \ + > "$RUNNER_TEMP/audit.json" || true + python3 pkg/contrib/check-lock-audit.py \ + --report "$RUNNER_TEMP/audit.json" \ + --tree "$RUNNER_TEMP/wasmtime" diff --git a/pkg/contrib/check-lock-audit.py b/pkg/contrib/check-lock-audit.py new file mode 100755 index 000000000..310ea1070 --- /dev/null +++ b/pkg/contrib/check-lock-audit.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Gate a `cargo audit --json` report on the pinned Wasmtime source lock. + +The archived Cargo.lock covers the whole Wasmtime workspace: the CLI, the +fuzzing targets and wasi-http are all in it, while Unit links one C API +library built from a subset of the features. A finding against a crate that +the library does not link is not a finding against Unit, so auditing the lock +as a whole reports far more than it can act on. + +The archive cannot be repaired either. It ships `.cargo/config.toml` with +`replace-with = "vendored-sources"` over a `vendor/` directory, so +`cargo update --precise` inside it is a no-op: the vendor set holds the pinned +versions and nothing else. + +So this script splits the report in two. A finding against a crate that the C +API links fails the job unless the allowlist accepts that exact version with a +reason. A finding outside the linked graph is reported and does not fail. + +The linked graph is computed, never declared: `cargo tree` resolves it offline +against the vendored archive, with the feature set that pkg/contrib builds. A +Wasmtime bump that starts linking an affected crate therefore turns the job red +on its own, which a hand-written "not linked" note would not. + +Exit codes: 0 clean, 1 a linked finding is not allowed, 2 configuration or +tooling failure. +""" +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +MAKEFILE = REPO / "pkg/contrib/src/wasmtime/Makefile" +ALLOWLIST = REPO / "pkg/contrib/wasmtime-lock-audit.allow.json" +FEATURES_CMAKE = Path("crates/c-api/cmake/features.cmake") +CMAKELISTS = Path("crates/c-api/CMakeLists.txt") + +PACKAGE_RE = re.compile(r"^(?P[\w.+-]+) v(?P[^\s]+)") +FEATURE_RE = re.compile( + r"^\s*feature\((?P[\w-]+)\s+(?PON|OFF)\)", re.MULTILINE +) +CMAKE_FLAG_RE = re.compile(r"-D(?PWASMTIME_[A-Z0-9_]+)=(?PON|OFF)") + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + sys.exit(2) + + +def rel(path: Path) -> Path: + """Shorten a path for a message, unless a test moved it out of the tree.""" + try: + return path.relative_to(REPO) + except ValueError: + return path + + +def cmake_name(feature: str) -> str: + return f"wasmtime_feature_{feature}".upper().replace("-", "_") + + +def read_cmake_flags() -> dict: + """Read the -DWASMTIME_* flags that pkg/contrib passes to cmake.""" + if not MAKEFILE.is_file(): + fail(f"{MAKEFILE} is missing") + return { + match.group("name"): match.group("value") == "ON" + for match in CMAKE_FLAG_RE.finditer(MAKEFILE.read_text()) + } + + +def read_features(tree: Path) -> list: + """Resolve the Cargo features of the C API library as cmake would.""" + source = tree / FEATURES_CMAKE + if not source.is_file(): + fail(f"{source} is missing; is {tree} an unpacked Wasmtime archive?") + + # The feature list below is only the whole story while cmake still starts + # from nothing and passes exactly what features.cmake collected. If a + # release changes either, the resolved graph shifts and findings move + # quietly in or out of it. + text = source.read_text() + if "--no-default-features" not in text: + fail(f"{FEATURES_CMAKE} no longer starts from --no-default-features") + + cmakelists = tree / CMAKELISTS + if not cmakelists.is_file(): + fail(f"{cmakelists} is missing") + if "${WASMTIME_FEATURES}" not in cmakelists.read_text(): + fail(f"{CMAKELISTS} no longer passes ${{WASMTIME_FEATURES}} to cargo") + + flags = read_cmake_flags() + disable_all = flags.get("WASMTIME_DISABLE_ALL_FEATURES", False) + + features = [] + declared = set() + for match in FEATURE_RE.finditer(text): + name = match.group("name") + declared.add(cmake_name(name)) + default = match.group("default") == "ON" and not disable_all + if flags.get(cmake_name(name), default): + features.append(name) + + if not features: + fail(f"no features parsed from {source}") + + unknown = set(flags) - declared - {"WASMTIME_DISABLE_ALL_FEATURES"} + if unknown: + fail( + f"{rel(MAKEFILE)} sets {', '.join(sorted(unknown))}, " + f"which {FEATURES_CMAKE} does not declare" + ) + + return features + + +def linked_packages(tree: Path) -> set: + """Return the (name, version) pairs the C API library links.""" + command = [ + "cargo", + "tree", + "--offline", + "--locked", + "--package=wasmtime-c-api", + "--edges=normal,build", + # Without this cargo resolves the host triple alone, and a crate + # linked only on another architecture would read as not linked. + "--target=all", + "--prefix=none", + "--no-default-features", + ] + command += [f"--features={feature}" for feature in read_features(tree)] + + try: + result = subprocess.run( + command, cwd=tree, capture_output=True, text=True, check=True + ) + except OSError as error: + fail(f"cannot run cargo tree: {error}") + except subprocess.CalledProcessError as error: + fail(f"cargo tree failed:\n{error.stderr.strip()}") + + packages = set() + for line in result.stdout.splitlines(): + match = PACKAGE_RE.match(line) + if match: + packages.add((match.group("name"), match.group("version"))) + + if not packages: + fail("cargo tree reported no packages") + + return packages + + +def read_findings(report: Path) -> list: + """Flatten a cargo audit report into (kind, id, package, version) tuples. + + A yanked crate carries no advisory, so the identifier is None there and + the allowlist has to key on the kind and the exact version instead. + """ + try: + data = json.loads(report.read_text()) + except OSError as error: + fail(f"cannot read {report}: {error}") + except json.JSONDecodeError as error: + fail(f"{report} is not valid JSON: {error}") + + if not isinstance(data, dict) or "vulnerabilities" not in data: + fail(f"{report} is not a cargo audit report") + + def package(entry: dict) -> tuple: + try: + return entry["package"]["name"], entry["package"]["version"] + except (KeyError, TypeError): + fail(f"{report} has an entry without a package name and version") + + findings = [] + for entry in data["vulnerabilities"].get("list", []): + advisory = entry.get("advisory") or {} + findings.append(("vulnerability", advisory.get("id"), *package(entry))) + + # Warnings are grouped by kind, and a kind this script has never seen is + # still a warning: iterate whatever the report carries. + for kind, entries in sorted(data.get("warnings", {}).items()): + for entry in entries: + advisory = entry.get("advisory") or {} + findings.append((kind, advisory.get("id"), *package(entry))) + + return findings + + +def read_allowlist(path: Path) -> dict: + try: + data = json.loads(path.read_text()) + except OSError as error: + fail(f"cannot read {path}: {error}") + except json.JSONDecodeError as error: + fail(f"{path} is not valid JSON: {error}") + + allowed = {} + for entry in data.get("allow", []): + missing = {"kind", "package", "version", "reason"} - set(entry) + if missing: + fail(f"{path} has an entry without {', '.join(sorted(missing))}") + key = (entry["kind"], entry.get("id"), entry["package"], entry["version"]) + if key in allowed: + fail(f"{path} lists {entry['package']} {entry['version']} twice") + allowed[key] = entry["reason"] + + return allowed + + +def describe(finding: tuple) -> str: + kind, identifier, package, version = finding + label = identifier or kind + return f"{package} {version}: {label}" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--report", required=True, type=Path, help="cargo audit --json output" + ) + parser.add_argument( + "--tree", type=Path, help="the unpacked Wasmtime source archive" + ) + parser.add_argument( + "--linked", type=Path, help="a cargo tree listing, instead of --tree" + ) + parser.add_argument("--allow", type=Path, default=ALLOWLIST) + args = parser.parse_args() + + if (args.tree is None) == (args.linked is None): + fail("pass either --tree or --linked") + + if args.linked: + packages = { + (match.group("name"), match.group("version")) + for line in args.linked.read_text().splitlines() + if (match := PACKAGE_RE.match(line)) + } + else: + packages = linked_packages(args.tree) + + findings = read_findings(args.report) + allowed = read_allowlist(args.allow) + + unlinked = [] + accepted = [] + denied = [] + for finding in findings: + _, _, package, version = finding + if (package, version) not in packages: + unlinked.append(finding) + elif finding in allowed: + accepted.append(finding) + else: + denied.append(finding) + + print(f"{len(packages)} packages linked by the C API library") + + for finding in unlinked: + print(f"not linked: {describe(finding)}") + + for finding in accepted: + print(f"accepted: {describe(finding)}") + print(f" {allowed[finding]}") + + # An advisory id is None for a yank, so sort on a string in its place. + stale = sorted(set(allowed) - set(accepted), key=lambda f: (f[0], f[1] or "", *f[2:])) + for finding in stale: + print(f"stale: {describe(finding)} is no longer reported as linked") + + for finding in denied: + print(f"error: {describe(finding)} is linked and not allowed") + + if denied: + print( + f"\n{len(denied)} linked finding(s) are not in " + f"{args.allow.name}. Bump the pin, or add an entry with the " + f"reason the build is not affected.", + file=sys.stderr, + ) + return 1 + + print(f"\n{len(findings)} finding(s), none linked and unaccounted for") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pkg/contrib/test_check_lock_audit.py b/pkg/contrib/test_check_lock_audit.py new file mode 100644 index 000000000..c665b3be7 --- /dev/null +++ b/pkg/contrib/test_check_lock_audit.py @@ -0,0 +1,348 @@ +"""Tests for check-lock-audit.py. + +These run without a Unit binary, so they live next to the script instead of in +test/, where conftest.py starts a server for every session. Run them with +`python3 -m pytest pkg/contrib/test_check_lock_audit.py`. +""" +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).with_name("check-lock-audit.py") + +LINKED = """\ +wasmtime-c-api v47.0.4 (/src/crates/c-api/artifact) +crossbeam-epoch v0.9.18 +chacha20 v0.10.0 +anyhow v1.0.100 +wasmtime v47.0.4 (/src/crates/wasmtime) +""" + +ALLOW = { + "allow": [ + { + "kind": "vulnerability", + "id": "RUSTSEC-2026-0204", + "package": "crossbeam-epoch", + "version": "0.9.18", + "reason": "not reached", + }, + { + "kind": "yanked", + "id": None, + "package": "chacha20", + "version": "0.10.0", + "reason": "a yank is not an advisory", + }, + ] +} + + +def vulnerability(identifier, package, version): + return { + "advisory": {"id": identifier}, + "package": {"name": package, "version": version}, + } + + +def report(vulnerabilities=(), warnings=None): + return { + "vulnerabilities": { + "count": len(vulnerabilities), + "list": list(vulnerabilities), + }, + "warnings": warnings or {}, + } + + +BASELINE = report( + vulnerabilities=[vulnerability("RUSTSEC-2026-0204", "crossbeam-epoch", "0.9.18")], + warnings={ + "yanked": [{"advisory": None, "package": {"name": "chacha20", "version": "0.10.0"}}], + "unmaintained": [ + { + "advisory": {"id": "RUSTSEC-2024-0436"}, + "package": {"name": "paste", "version": "1.0.7"}, + } + ], + }, +) + + +@pytest.fixture +def run(tmp_path): + def call(report_text, allow=ALLOW, linked_text=LINKED): + linked = tmp_path / "linked.txt" + linked.write_text(linked_text) + report_file = tmp_path / "audit.json" + if report_text is not None: + report_file.write_text(report_text) + allow_file = tmp_path / "allow.json" + allow_file.write_text(json.dumps(allow)) + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--report", + str(report_file), + "--linked", + str(linked), + "--allow", + str(allow_file), + ], + capture_output=True, + text=True, + ) + + return call + + +def test_allowlisted_findings_pass(run): + result = run(json.dumps(BASELINE)) + assert result.returncode == 0, result.stdout + result.stderr + assert "accepted: crossbeam-epoch 0.9.18" in result.stdout + assert "accepted: chacha20 0.10.0: yanked" in result.stdout + + +def test_unlinked_finding_is_reported_but_passes(run): + result = run(json.dumps(BASELINE)) + assert result.returncode == 0 + assert "not linked: paste 1.0.7: RUSTSEC-2024-0436" in result.stdout + + +def test_unknown_linked_vulnerability_fails(run): + data = report( + vulnerabilities=[ + vulnerability("RUSTSEC-2026-0204", "crossbeam-epoch", "0.9.18"), + vulnerability("RUSTSEC-2099-0001", "anyhow", "1.0.100"), + ] + ) + result = run(json.dumps(data)) + assert result.returncode == 1 + assert "error: anyhow 1.0.100: RUSTSEC-2099-0001 is linked" in result.stdout + + +def test_unknown_linked_yank_fails(run): + data = report( + warnings={ + "yanked": [ + {"advisory": None, "package": {"name": "anyhow", "version": "1.0.100"}} + ] + } + ) + result = run(json.dumps(data)) + assert result.returncode == 1 + assert "error: anyhow 1.0.100: yanked is linked" in result.stdout + + +def test_an_unseen_warning_kind_is_still_gated(run): + data = report( + warnings={ + "unsound": [ + { + "advisory": {"id": "RUSTSEC-2099-0002"}, + "package": {"name": "anyhow", "version": "1.0.100"}, + } + ] + } + ) + result = run(json.dumps(data)) + assert result.returncode == 1 + assert "RUSTSEC-2099-0002" in result.stdout + + +def test_the_allowlist_version_is_part_of_the_key(run): + # An acceptance is written against one version; the next pin has to earn + # its own entry rather than inherit this one. + linked = LINKED + "crossbeam-epoch v0.9.20\n" + data = report( + vulnerabilities=[ + vulnerability("RUSTSEC-2026-0204", "crossbeam-epoch", "0.9.20") + ] + ) + result = run(json.dumps(data), linked_text=linked) + assert result.returncode == 1 + assert "error: crossbeam-epoch 0.9.20: RUSTSEC-2026-0204" in result.stdout + + +def test_a_stale_entry_warns_without_failing(run): + data = report( + vulnerabilities=[vulnerability("RUSTSEC-2026-0204", "crossbeam-epoch", "0.9.18")] + ) + result = run(json.dumps(data)) + assert result.returncode == 0 + assert "stale: chacha20 0.10.0: yanked is no longer reported" in result.stdout + + +def test_malformed_report_fails(run): + result = run("{not json") + assert result.returncode == 2 + assert "not valid JSON" in result.stderr + + +def test_an_empty_report_fails(run): + result = run("{}") + assert result.returncode == 2 + assert "not a cargo audit report" in result.stderr + + +def test_a_missing_report_fails(run): + result = run(None) + assert result.returncode == 2 + assert "cannot read" in result.stderr + + +def test_an_allowlist_entry_needs_a_reason(run): + allow = {"allow": [{"kind": "yanked", "package": "chacha20", "version": "0.10.0"}]} + result = run(json.dumps(BASELINE), allow) + assert result.returncode == 2 + assert "without reason" in result.stderr + + +def module(): + spec = importlib.util.spec_from_file_location("check_lock_audit", SCRIPT) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +def archive(tmp_path, check, features_cmake): + """Lay out the parts of an unpacked archive that read_features() reads.""" + cmake = tmp_path / check.FEATURES_CMAKE + cmake.parent.mkdir(parents=True, exist_ok=True) + cmake.write_text(features_cmake) + cmakelists = tmp_path / check.CMAKELISTS + cmakelists.write_text("${WASMTIME_CARGO_BINARY} build ${WASMTIME_FEATURES}\n") + return tmp_path + + +def test_the_feature_set_follows_the_cmake_override(tmp_path, monkeypatch): + check = module() + makefile = tmp_path / "Makefile" + makefile.write_text("-DWASMTIME_FEATURE_WASI_HTTP=OFF \\\n") + monkeypatch.setattr(check, "MAKEFILE", makefile) + tree = archive( + tmp_path, + check, + 'set(WASMTIME_FEATURES "--no-default-features")\n' + "feature(wasi ON)\nfeature(wasi-http ON)\nfeature(all-arch OFF)\n", + ) + assert check.read_features(tree) == ["wasi"] + + +@pytest.mark.parametrize( + "features_cmake, cmakelists", + [ + ('set(WASMTIME_FEATURES "")\nfeature(wasi ON)\n', None), + (None, "${WASMTIME_CARGO_BINARY} build\n"), + ], +) +def test_a_changed_cargo_invocation_fails( + tmp_path, monkeypatch, features_cmake, cmakelists +): + """The reconstruction is only valid while cmake drives cargo this way.""" + check = module() + makefile = tmp_path / "Makefile" + makefile.write_text("") + monkeypatch.setattr(check, "MAKEFILE", makefile) + tree = archive( + tmp_path, + check, + features_cmake + or 'set(WASMTIME_FEATURES "--no-default-features")\nfeature(wasi ON)\n', + ) + if cmakelists: + (tree / check.CMAKELISTS).write_text(cmakelists) + with pytest.raises(SystemExit) as error: + check.read_features(tree) + assert error.value.code == 2 + + +def test_an_undeclared_cmake_flag_fails(tmp_path, monkeypatch): + check = module() + makefile = tmp_path / "Makefile" + makefile.write_text("-DWASMTIME_FEATURE_NO_SUCH_THING=OFF\n") + monkeypatch.setattr(check, "MAKEFILE", makefile) + tree = archive( + tmp_path, + check, + 'set(WASMTIME_FEATURES "--no-default-features")\nfeature(wasi ON)\n', + ) + with pytest.raises(SystemExit) as error: + check.read_features(tree) + assert error.value.code == 2 + + +def test_a_cargo_tree_listing_parses(tmp_path): + """The shapes cargo tree --prefix=none actually prints.""" + check = module() + listing = tmp_path / "tree.txt" + listing.write_text( + "wasmtime v47.0.4 (/src/crates/wasmtime)\n" + "anyhow v1.0.100\n" + "anyhow v1.0.100 (*)\n" + "serde_derive v1.0.228 (proc-macro)\n" + "wasi v0.11.0+wasi-snapshot-preview1\n" + "\n" + ) + found = { + (match.group("name"), match.group("version")) + for line in listing.read_text().splitlines() + if (match := check.PACKAGE_RE.match(line)) + } + assert found == { + ("wasmtime", "47.0.4"), + ("anyhow", "1.0.100"), + ("serde_derive", "1.0.228"), + ("wasi", "0.11.0+wasi-snapshot-preview1"), + } + + +def test_a_duplicate_allowlist_entry_fails(run): + entry = { + "kind": "yanked", + "id": None, + "package": "chacha20", + "version": "0.10.0", + "reason": "first", + } + allow = {"allow": [entry, dict(entry, reason="second")]} + result = run(json.dumps(BASELINE), allow) + assert result.returncode == 2 + assert "twice" in result.stderr + + +def test_a_warning_without_a_package_fails(run): + data = report(warnings={"yanked": [{"advisory": None}]}) + result = run(json.dumps(data)) + assert result.returncode == 2 + assert "without a package name and version" in result.stderr + + +def test_stale_entries_sort_with_and_without_an_advisory_id(run): + # Sorting the stale list used to compare None against a string. + allow = { + "allow": [ + { + "kind": "yanked", + "id": None, + "package": "aaa", + "version": "1.0.0", + "reason": "no advisory", + }, + { + "kind": "yanked", + "id": "RUSTSEC-2099-0003", + "package": "bbb", + "version": "2.0.0", + "reason": "with advisory", + }, + ] + } + result = run(json.dumps(report()), allow) + assert result.returncode == 0 + assert result.stdout.count("stale:") == 2 diff --git a/pkg/contrib/wasmtime-lock-audit.allow.json b/pkg/contrib/wasmtime-lock-audit.allow.json new file mode 100644 index 000000000..f338f7c16 --- /dev/null +++ b/pkg/contrib/wasmtime-lock-audit.allow.json @@ -0,0 +1,26 @@ +{ + "comment": [ + "Findings against crates that the Wasmtime C API library links, and that", + "pkg/contrib cannot resolve by bumping the pin. check-lock-audit.py fails", + "the audit on any other linked finding. A finding outside the linked graph", + "needs no entry here; it is reported and ignored.", + "The version is part of the key, so a pin bump forces this file to be", + "reviewed again rather than carrying an acceptance forward silently." + ], + "allow": [ + { + "kind": "vulnerability", + "id": "RUSTSEC-2026-0204", + "package": "crossbeam-epoch", + "version": "0.9.18", + "reason": "Linked through rayon, which wasmtime pulls in for the parallel-compilation feature. The bug is an invalid pointer dereference in the fmt::Pointer impl of Atomic and Shared; neither wasmtime nor Unit formats an epoch pointer, so no call site reaches it. The archive is vendored and cannot be relocked, and wasmtime 48.x already resolves crossbeam-epoch 0.9.20, so the next pin bump clears this." + }, + { + "kind": "yanked", + "id": null, + "package": "chacha20", + "version": "0.10.0", + "reason": "Linked through rand, which wasmtime-wasi uses to seed guest randomness. A yank is not an advisory: no RustSec record applies to this version. 0.10.1 is yanked as well and 0.10.2 is clean, but the archive is vendored, so the version moves only when the pin does." + } + ] +} diff --git a/pkg/eol.json b/pkg/eol.json index 3affc4aff..0e94bfe15 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -131,7 +131,7 @@ "role": "WebAssembly runtime built by pkg/contrib and linked by the wasm module (pkg/contrib/src/wasmtime/version, WASMTIME_VERSION)", "version": "47.0.4", "eol": null, - "eol_note": "Fetched as a source tarball from packages.freeunit.org, so cargo audit never sees it: it reads Cargo.lock files only. Covered by the Audit (cargo) pins job, which queries OSV for the version in pkg/contrib/src/wasmtime/version, so an advisory here arrives as a red job rather than silence. Same 47.0.4 as the wasmtime_crate pin; the 43.0.1 bump closed RUSTSEC-2026-0269 (filesystem sandbox escape), -0222, -0114, and wasmtime-wasi -0149, -0182 and -0188 (WASI path and file-permission bypasses). Tracked in freeunitorg/freeunit#400.", + "eol_note": "Fetched as a source tarball from packages.freeunit.org, so its Cargo.lock is not among the repository locks. The Audit (cargo) pins job verifies the tarball digest, queries OSV for the direct source-package pin, and audits the archived Cargo.lock, so an advisory here arrives as a red job rather than silence. That lock is the whole Wasmtime workspace, so pkg/contrib/check-lock-audit.py fails it only for a crate the C API library links, with pkg/contrib/wasmtime-lock-audit.allow.json recording the accepted ones; the archive is vendored, so a finding moves only when the pin does. Same 47.0.4 as the wasmtime_crate pin; the 43.0.1 bump closed RUSTSEC-2026-0269 (filesystem sandbox escape), -0222, -0114, and wasmtime-wasi -0149, -0182 and -0188 (WASI path and file-permission bypasses). Tracked in freeunitorg/freeunit#400.", "source": "https://github.com/bytecodealliance/wasmtime/releases" }, "wasmtime_crate": { From 3d707b73596f58f8bc6ab3f7e43faf6ade44fc94 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 21 Sep 2026 15:44:07 +0200 Subject: [PATCH 230/391] var: drop the write-only variable index clang 23 adds -Wunused-but-set-global, a subgroup of the -Wunused-but-set-variable that -Wall already enables, and configure adds -Werror. It fails the build on nxt_vars: src/nxt_var.c:75:29: error: variable 'nxt_vars' set but not used The diagnostic is right. nxt_vars is a file-static pointer that nxt_var_index_init() fills and nothing reads; the history of nxt_var.c never had a reader. So the whole function is dead at startup: it allocates a 64-byte aligned array of every registered variable, copies the hash into it, stores the pointer and returns. nxt_var_count exists only to size that array. Remove the index, its counter, the prototype and the call from nxt_runtime_create(). The remaining nxt_var_register() failure path is unchanged. No behaviour changes; the startup allocation goes away. Co-Authored-By: Claude Fable 5.1 --- src/nxt_runtime.c | 4 ---- src/nxt_var.c | 30 ------------------------------ src/nxt_var.h | 1 - 3 files changed, 35 deletions(-) diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 0f87082b7..0c38247a1 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -134,10 +134,6 @@ nxt_runtime_create(nxt_task_t *task) goto fail; } - if (nxt_slow_path(nxt_var_index_init() != NXT_OK)) { - goto fail; - } - nxt_work_queue_add(&task->thread->engine->fast_work_queue, nxt_runtime_start, task, rt, NULL); diff --git a/src/nxt_var.c b/src/nxt_var.c index bdb7f17f0..4851bedbb 100644 --- a/src/nxt_var.c +++ b/src/nxt_var.c @@ -70,9 +70,6 @@ static const nxt_lvlhsh_proto_t nxt_var_cache_proto nxt_aligned(64) = { static nxt_lvlhsh_t nxt_var_hash; -static uint32_t nxt_var_count; - -static nxt_var_decl_t **nxt_vars; static nxt_int_t @@ -286,33 +283,6 @@ nxt_var_register(nxt_var_decl_t *decl, size_t n) } } - nxt_var_count += n; - - return NXT_OK; -} - - -nxt_int_t -nxt_var_index_init(void) -{ - nxt_uint_t i; - nxt_var_decl_t *decl, **vars; - nxt_lvlhsh_each_t lhe; - - vars = nxt_memalign(64, nxt_var_count * sizeof(nxt_var_decl_t *)); - if (vars == NULL) { - return NXT_ERROR; - } - - nxt_lvlhsh_each_init(&lhe, &nxt_var_hash_proto); - - for (i = 0; i < nxt_var_count; i++) { - decl = nxt_lvlhsh_each(&nxt_var_hash, &lhe); - vars[i] = decl; - } - - nxt_vars = vars; - return NXT_OK; } diff --git a/src/nxt_var.h b/src/nxt_var.h index 013654420..b84ca212e 100644 --- a/src/nxt_var.h +++ b/src/nxt_var.h @@ -49,7 +49,6 @@ typedef struct { nxt_int_t nxt_var_register(nxt_var_decl_t *decl, size_t n); -nxt_int_t nxt_var_index_init(void); nxt_var_field_t *nxt_var_field_get(nxt_array_t *fields, uint16_t index); nxt_var_field_t *nxt_var_field_new(nxt_mp_t *mp, nxt_str_t *name, From bbab15932c19ebe9d6f6507fbfadf08a2fd28c0d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 01:37:00 +0000 Subject: [PATCH 231/391] Fix router crash on object-form "compressors" (#431) nxt_conf_vldt_compressors() handed the object branch of "compressors" to nxt_conf_vldt_object_iterator(), which casts its "data" argument to a single validator function and calls it. The argument passed was nxt_conf_vldt_compressor_members[], a descriptor table, not a function, so the router made a wild call into whatever sat at the head of that array and crashed validating an otherwise ordinary config. Route the object form through nxt_conf_vldt_object() instead, which is what actually consumes a descriptor table -- the same helper nxt_conf_vldt_compression() already uses to validate one array element. test/test_conf_compressors_object.py reproduces the crash against an unfixed binary (the control socket goes silent and the router restarts) and checks the object form now validates, rejects a bad encoding without dying, and leaves the router able to keep validating afterwards. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA (cherry picked from commit 3c784b5328a9ffe7e7d6672499f78c2ac8310880) --- src/nxt_conf_validation.c | 3 +-- test/test_conf_compressors_object.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 test/test_conf_compressors_object.py diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index dae8e3854..b1b7b728d 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -2854,8 +2854,7 @@ nxt_conf_vldt_compressors(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, /* NXT_CONF_OBJECT */ - return nxt_conf_vldt_object_iterator(vldt, value, - &nxt_conf_vldt_compressor_members); + return nxt_conf_vldt_object(vldt, value, nxt_conf_vldt_compressor_members); } diff --git a/test/test_conf_compressors_object.py b/test/test_conf_compressors_object.py new file mode 100644 index 000000000..c4677a51f --- /dev/null +++ b/test/test_conf_compressors_object.py @@ -0,0 +1,27 @@ +"""#431: the object form of "compressors" crashed the router's validator.""" + +import pytest + +from unit.control import Control + +client = Control() + + +@pytest.mark.parametrize( + 'compressor, result', + [ + ({"encoding": "identity", "min_length": 10}, 'success'), + ({"encoding": "not-a-real-encoding"}, 'error'), + ], +) +def test_compressors_object_form(compressor, result): + base = { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 200}}], + } + compression = {"types": ["text/css"], "compressors": compressor} + + assert result in client.conf( + {"settings": {"http": {"compression": compression}}, **base} + ) + assert 'success' in client.conf(base), 'the router is still alive' From 9d4ad493023cc7e487ad45c0bde3d737584f4593 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 17:26:56 +0000 Subject: [PATCH 232/391] router: stop the request deadline at a WebSocket upgrade (#422) "limits": {"timeout"} bounds how long a worker takes to answer a request. The 101 arrives as a non-last message, so nxt_router_response_ready_handler() re-armed the deadline for it, and again for every later non-last message -- which, after the upgrade, is every frame the application sends. Nothing disabled it, so a session quiet for longer than the timeout reached nxt_router_app_timeout(), which answered 503 into an upgraded connection and unlinked the rpc data. What the client sees, established by the new test: the connection is closed about one timeout after the handshake -- recv() returns EOF, no close frame, no HTTP bytes. The worker logs "active request on ctx quit/free" as the session is torn down under it. The deadline is now disabled where the upgrade takes effect (beside r->state = &nxt_http_websocket, after the 101 has re-armed it), and the re-arm for non-last messages skips an upgraded request. A handshake still in flight, or one that failed, keeps its deadline. Test: test/test_asgi_websockets_timeout.py (ASGI mirror app, limits.timeout 1): stays quiet for 3 s, then expects its frame echoed. Red before the fix ("nothing arrives on a quiet session: b''", EOF at ~1 s); green after. test_asgi_websockets.py and the C router websocket / app-timeout tests still pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA (cherry picked from commit 8e1234955588a92e144559054754c5459d6f1b52) --- src/nxt_router.c | 14 +++++++- test/test_asgi_websockets_timeout.py | 48 ++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 test/test_asgi_websockets_timeout.py diff --git a/src/nxt_router.c b/src/nxt_router.c index 34ae7309f..43568fe49 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5415,7 +5415,12 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_request_rpc_data_unlink(task, req_rpc_data); } else { - if (app->timeout != 0) { + /* + * The deadline bounds the worker's answer, and an upgraded + * WebSocket has had its answer: the frames that follow are a + * session, however quiet (#422). + */ + if (app->timeout != 0 && r->state != &nxt_http_websocket) { r->timer.handler = nxt_router_app_timeout; r->timer_data = req_rpc_data; nxt_timer_add(task->thread->engine, &r->timer, app->timeout); @@ -5591,6 +5596,13 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, r->state = &nxt_http_websocket; + /* + * The 101 arrived as a non-last message, which re-armed the + * request deadline above; the upgrade ends the request, so the + * deadline goes with it (#422). + */ + nxt_timer_disable(task->thread->engine, &r->timer); + } else { r->state = &nxt_http_request_send_state; } diff --git a/test/test_asgi_websockets_timeout.py b/test/test_asgi_websockets_timeout.py new file mode 100644 index 000000000..a78c8ad4e --- /dev/null +++ b/test/test_asgi_websockets_timeout.py @@ -0,0 +1,48 @@ +"""limits.timeout must not end a quiet, established WebSocket (#422).""" + +from packaging import version + +from unit.applications.lang.python import ApplicationPython +from unit.applications.websockets import ApplicationWebsocket + +prerequisites = { + 'modules': {'python': lambda v: version.parse(v) >= version.parse('3.5')} +} + +client = ApplicationPython(load_module='asgi') +ws = ApplicationWebsocket() + + +def test_asgi_websockets_timeout_quiet_session(skip_alert): + skip_alert(r'socket close\(\d+\) failed') + + assert 'success' in client.conf( + {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' + ), 'clear keepalive_interval' + + client.load('websockets/mirror', limits={'timeout': 1}) + + resp, sock, _ = ws.upgrade() + assert resp['status'] == 101, 'upgrade' + + # frame_read() would spin on EOF. + sock.settimeout(3) + + try: + early = sock.recv(4096) + except TimeoutError: + early = None + + assert early is None, f'nothing arrives on a quiet session: {early!r}' + + ws.frame_write(sock, ws.OP_TEXT, 'still here') + frame = ws.frame_read(sock, read_timeout=2) + + assert frame['opcode'] == ws.OP_TEXT, 'session survived the deadline' + assert frame['data'].decode('utf-8') == 'still here', 'mirror' + + ws.frame_write(sock, ws.OP_CLOSE, ws.serialize_close()) + frame = ws.frame_read(sock) + assert frame['opcode'] == ws.OP_CLOSE, 'close' + + sock.close() From 841b6d012fc5fa14ae76d8361072336175714807 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 07:45:45 +0000 Subject: [PATCH 233/391] java: report a binary close reason for an oversized binary message WsSession.onMessage(ByteBuffer, boolean) reused the text message's "wsFrame.textMessageTooBig" key for a fragmented binary message that overflowed the session's binary buffer. The close code (1009) was right; only the reason string named the wrong message kind, which made a binary overflow and a text overflow indistinguishable from the client side. Adds "wsFrame.binaryMessageTooBig" and uses it on the binary path; the unfragmented binary path short-circuits earlier and never reaches this check either way. Adds test/java/websockets_binary_toobig, a mirror app with a 1 KiB binary buffer, so the fragmented-overflow branch can be exercised with two small frames rather than the 16 MiB the shared websockets_mirror app now needs. Closes #435 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA (cherry picked from commit 81263559cb3f7af8ca484d8968ed0bd1c0dc97e5) --- .../unit/websocket/LocalStrings.properties | 1 + src/java/nginx/unit/websocket/WsSession.java | 2 +- test/java/websockets_binary_toobig/app.java | 58 +++++++++++++++++++ test/test_java_websockets.py | 21 +++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 test/java/websockets_binary_toobig/app.java diff --git a/src/java/nginx/unit/websocket/LocalStrings.properties b/src/java/nginx/unit/websocket/LocalStrings.properties index aeafe0823..a814a11c0 100644 --- a/src/java/nginx/unit/websocket/LocalStrings.properties +++ b/src/java/nginx/unit/websocket/LocalStrings.properties @@ -57,6 +57,7 @@ wsFrame.alreadySuspended=Message receiving has already been suspended. wsFrame.bufferTooSmall=No async message support and buffer too small. Buffer size: [{0}], Message size: [{1}] wsFrame.byteToLongFail=Too many bytes ([{0}]) were provided to be converted into a long wsFrame.closed=New frame received after a close control frame +wsFrame.binaryMessageTooBig=The binary message was too big for the output buffer and the endpoint does not support partial messages wsFrame.controlFragmented=A fragmented control frame was received but control frames may not be fragmented wsFrame.controlPayloadTooBig=A control frame was sent with a payload of size [{0}] which is larger than the maximum permitted of 125 bytes wsFrame.controlNoFin=A control frame was sent that did not have the fin bit set. Control frames are not permitted to use continuation frames. diff --git a/src/java/nginx/unit/websocket/WsSession.java b/src/java/nginx/unit/websocket/WsSession.java index b654eb372..3bd3064c0 100644 --- a/src/java/nginx/unit/websocket/WsSession.java +++ b/src/java/nginx/unit/websocket/WsSession.java @@ -989,7 +989,7 @@ public void onMessage(ByteBuffer buf, boolean last) if (binaryBuffer.remaining() < buf.remaining()) { throw new WsIOException(new CloseReason( CloseCodes.TOO_BIG, - sm.getString("wsFrame.textMessageTooBig"))); + sm.getString("wsFrame.binaryMessageTooBig"))); } binaryBuffer.put(buf); diff --git a/test/java/websockets_binary_toobig/app.java b/test/java/websockets_binary_toobig/app.java new file mode 100644 index 000000000..7f94f0bc6 --- /dev/null +++ b/test/java/websockets_binary_toobig/app.java @@ -0,0 +1,58 @@ +import java.io.IOException; +import java.nio.ByteBuffer; + +import javax.websocket.OnMessage; +import javax.websocket.OnOpen; +import javax.websocket.PongMessage; +import javax.websocket.Session; +import javax.websocket.server.ServerEndpoint; + +/* + * A minimal mirror app with a small binary buffer, so a fragmented binary + * message that overflows it can be exercised with a handful of bytes + * instead of the 16 MiB the shared websockets_mirror app now requires + * (see issue #435). + */ +@ServerEndpoint("/") +public class app { + + @OnOpen + public void onOpen(Session session) { + session.setMaxBinaryMessageBufferSize(1024); + } + + @OnMessage + public void echoTextMessage(Session session, String msg) { + try { + if (session.isOpen()) { + session.getBasicRemote().sendText(msg, true); + } + } catch (IOException e) { + try { + session.close(); + } catch (IOException e1) { + // Ignore + } + } + } + + @OnMessage + public void echoBinaryMessage(Session session, ByteBuffer bb) { + try { + if (session.isOpen()) { + session.getBasicRemote().sendBinary(bb, true); + } + } catch (IOException e) { + try { + session.close(); + } catch (IOException e1) { + // Ignore + } + } + } + + @OnMessage + public void echoPongMessage(PongMessage pm) { + // NO-OP + } +} diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index fe0bb1bad..a70bf999c 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -1412,3 +1412,24 @@ def test_java_websockets_keepalive_interval(): check_frame(frame, True, ws.OP_PING, '') # PING frame sock.close() + + +def test_java_websockets_binary_message_too_big_reason(): + # A fragmented binary message over the session's binary buffer used to + # report the text-message close reason (issue #435); an unfragmented + # binary message never reaches the check, so this needs two frames. + client.load('websockets_binary_toobig') + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_BINARY, '*' * 600, fin=False) + ws.frame_write(sock, ws.OP_CONT, '*' * 600, fin=True) + + frame = ws.frame_read(sock) + + assert frame['opcode'] == ws.OP_CLOSE, 'close opcode' + assert frame['code'] == 1009, 'close code' + assert 'text message' not in frame['reason'], 'reason names the wrong kind' + assert 'binary' in frame['reason'], 'reason names the binary message' + + sock.close() From 797de61bf26da30ae905cab9d876e9fcb5a2f0bd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 19:56:21 +0000 Subject: [PATCH 234/391] feat(cert): check the key at upload and reserve dot names A PEM bundle was accepted when it had one private key and at least one certificate. Nothing checked that the key belongs to the first certificate. A wrong key was stored, and it failed later, when the router built the SSL context for a listener that named the bundle. Every reconfiguration of that listener then failed. Check the pair at upload with X509_check_private_key(). A bundle whose key does not match is refused with 400 "Invalid certificate.", and nothing is stored. Names that start with "." are now reserved. The control API refuses them with 400 "Invalid certificate name.", and the store skips such files at startup. The atomic in-place replace that follows keeps its temporary file under such a name, so a user cannot create or collide with it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- CHANGES | 7 +++++++ docs/changes.xml | 10 ++++++++++ docs/unit-openapi.yaml | 7 ++++++- src/nxt_cert.c | 14 +++++++++++++- src/nxt_controller.c | 3 ++- test/test_tls.py | 37 ++++++++++++++++++++++--------------- 6 files changed, 60 insertions(+), 18 deletions(-) diff --git a/CHANGES b/CHANGES index f27e22bd0..55974e94a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Change: a certificate bundle whose private key does not belong to + its first certificate is now refused at upload with 400 "Invalid + certificate.". Before, the bundle was stored, and the mismatch + broke the next reconfiguration of every listener that named it. + Certificate names that start with "." are reserved for the store's + own files, and files with such names are ignored at startup. + *) Bugfix: an application worker that served a WebSocket upgrade never became idle again. The router counts a session on the worker and keeps a worker with a session out of the idle queues, but nothing diff --git a/docs/changes.xml b/docs/changes.xml index de2b497d6..429d0d38d 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,16 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +a certificate bundle whose private key does not belong to its first +certificate is now refused at upload with 400 "Invalid certificate.". Before, +the bundle was stored, and the mismatch broke the next reconfiguration of every +listener that named it. Certificate names that start with "." are reserved +for the store's own files, and files with such names are ignored at startup. + + + an application worker that served a WebSocket upgrade never became idle again. diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 0a1a20123..4492593df 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -114,7 +114,12 @@ paths: operationId: putCertBundle summary: "Create or overwrite the actual certificate bundle" description: "Creates or overwrites the [stored certificate bundle] - (https://unit.nginx.org/certificates/) in Unit." + (https://unit.nginx.org/certificates/) in Unit. The body is a PEM + bundle: the server certificate first, then its chain, then the + private key. A bundle whose key does not belong to the first + certificate is refused with 400 `Invalid certificate.`; a name that + starts with `.` is reserved and refused with 400 + `Invalid certificate name.`." tags: - certificates diff --git a/src/nxt_cert.c b/src/nxt_cert.c index b8c3df607..9989fddad 100644 --- a/src/nxt_cert.c +++ b/src/nxt_cert.c @@ -324,6 +324,17 @@ nxt_cert_bio(nxt_task_t *task, BIO *bio) goto fail; } + /* + * The leaf comes first in the bundle. A key that belongs to another + * certificate is refused here, at upload, instead of at the next + * reconfiguration of every listener that names the bundle. + */ + if (X509_check_private_key(cert->chain[0], cert->key) != 1) { + nxt_openssl_log_error(task, NXT_LOG_ALERT, + "certificate and private key do not match"); + goto fail; + } + return cert; fail: @@ -965,7 +976,8 @@ nxt_cert_store_load(nxt_task_t *task, nxt_mp_t *mp) name.length = nxt_strlen(de->d_name); name.start = (u_char *) de->d_name; - if (nxt_str_eq(&name, ".", 1) || nxt_str_eq(&name, "..", 2)) { + /* ".", "..", and the names the store keeps for itself. */ + if (name.start[0] == '.') { continue; } diff --git a/src/nxt_controller.c b/src/nxt_controller.c index ce6dff6e0..e7c2bb0d9 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -1922,7 +1922,8 @@ nxt_controller_process_cert(nxt_task_t *task, return; } - if (name.length == 0 || path != NULL) { + /* Names starting with "." are reserved for the store's own files. */ + if (name.length == 0 || path != NULL || name.start[0] == '.') { goto invalid_name; } diff --git a/test/test_tls.py b/test/test_tls.py index ce605684c..6609dcce7 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -186,28 +186,35 @@ def test_tls_certificate_update(): def test_tls_certificate_key_incorrect(skip_alert): - skip_alert(r'nxt_openssl_chain_file\(\) failed', r'failed to apply new conf') + skip_alert(r'certificate and private key do not match') client.load('empty') client.certificate('first', False) client.certificate('second', False) - # A structurally valid bundle (a private key + a certificate) is stored - # fine -- the cert store only checks PEM structure. The key/cert - # *mismatch* is caught later, when the router builds the listener's - # SSL_CTX (SSL_CTX_check_private_key), rather than being left to fail at - # the first handshake. - assert 'success' in client.certificate_load( + # The bundle is structurally valid (a private key + a certificate), but + # the key belongs to another certificate. The store refuses it at + # upload, so a listener can never name it. + assert 'error' in client.certificate_load( 'first', 'second' - ), 'mismatched bundle stored' - - # pass at the existing app so the mismatched cert is the only reason - # the listener config can be rejected. - assert 'error' in client.conf( - {"pass": "applications/empty", "tls": {"certificate": 'first'}}, - 'listeners/*:8080', - ), 'mismatched key/cert rejected at listener config' + ), 'mismatched bundle refused' + + assert 'error' in client.conf_get('/certificates/first'), 'not stored' + + +def test_tls_certificate_dot_name(): + client.load('empty') + + client.certificate('default', False) + + # Names starting with "." are reserved for the store's own files. + for name in ['.', '..', '.default', '.default.tmp']: + assert 'error' in client.conf( + b'', f'/certificates/{name}' + ), f'dot name {name}' + + assert 'success' in client.certificate_load('default'), 'plain name' def test_tls_certificate_change(): From 23d7a39eb5413babd73bd26ade5030d6f789788c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 11:11:02 +0000 Subject: [PATCH 235/391] docs(cert): note what an upgrade does to a stored bundle with a bad key The key check also runs when the store is loaded at startup. A bundle that was stored before the upgrade with a key that does not match is then not loaded: an alert, and it leaves /certificates. The file stays in the state directory, and the API cannot delete a name it does not know. Say so in the changelog entry. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- CHANGES | 4 ++++ docs/changes.xml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/CHANGES b/CHANGES index 55974e94a..182befd2c 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,10 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 broke the next reconfiguration of every listener that named it. Certificate names that start with "." are reserved for the store's own files, and files with such names are ignored at startup. + After an upgrade, a bundle already stored with a mismatched key is + not loaded (an alert at startup) and leaves /certificates; its + file stays in the state directory and cannot be deleted through + the API, so remove it by hand. *) Bugfix: an application worker that served a WebSocket upgrade never became idle again. The router counts a session on the worker and diff --git a/docs/changes.xml b/docs/changes.xml index 429d0d38d..5d23698ed 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -17,6 +17,9 @@ certificate is now refused at upload with 400 "Invalid certificate.". Before, the bundle was stored, and the mismatch broke the next reconfiguration of every listener that named it. Certificate names that start with "." are reserved for the store's own files, and files with such names are ignored at startup. +After an upgrade, a bundle already stored with a mismatched key is not loaded +(an alert at startup) and leaves /certificates; its file stays in the state +directory and cannot be deleted through the API, so remove it by hand. From c66f4596d285cc111d2fc93b53b1a9e45f3baf43 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 10:43:40 +0000 Subject: [PATCH 236/391] test(http): free the lvlhsh tables the parse test builds nxt_http_parse_test() calls nxt_http_fields_hash() and nxt_http_fields_hash_collisions() three times to build lvlhsh tables for its collision count and its parser benchmark. Both functions use nxt_http_fields_hash_proto, which allocates lvlhsh buckets with malloc, not from a pool, and the test never freed any of the three tables it built this way. Under AddressSanitizer with leak detection on, this showed up as about 20 KB of "leaked" lvlhsh buckets and levels, all attributed to nxt_http_parse.c:1210 and :1254 through this test. Add nxt_http_parse_test_hash_destroy(), which drains a hash with nxt_lvlhsh_retrieve() the same way nxt_lvlhsh_test() already tears its own table down, and call it after each of the three hashes is done with, including on the early-return paths. Co-Authored-By: Claude Opus 5.5 --- src/test/nxt_http_parse_test.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/test/nxt_http_parse_test.c b/src/test/nxt_http_parse_test.c index 520bd4184..6c81c0b5e 100644 --- a/src/test/nxt_http_parse_test.c +++ b/src/test/nxt_http_parse_test.c @@ -64,6 +64,7 @@ static nxt_int_t nxt_http_parse_test_run(nxt_http_request_parse_t *rp, nxt_str_t *request); static nxt_int_t nxt_http_parse_test_bench(nxt_thread_t *thr, nxt_str_t *request, nxt_lvlhsh_t *hash, const char *name, nxt_uint_t n); +static void nxt_http_parse_test_hash_destroy(nxt_lvlhsh_t *hash); static nxt_int_t nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, nxt_http_parse_test_data_t *data, nxt_str_t *request, nxt_log_t *log); @@ -726,6 +727,24 @@ static nxt_str_t nxt_http_test_big_request = nxt_string( ); +/* + * nxt_http_fields_hash() and nxt_http_fields_hash_collisions() build a + * lvlhsh with the malloc-based nxt_http_fields_hash_proto (pool is NULL). + * The test drives them directly, so it must drain the hash itself here, + * the same way nxt_lvlhsh_retrieve() is used to tear down a lvlhsh in + * nxt_lvlhsh_test(), otherwise every bucket and level it allocated leaks. + */ +static void +nxt_http_parse_test_hash_destroy(nxt_lvlhsh_t *hash) +{ + while (nxt_lvlhsh_retrieve(hash, &nxt_http_fields_hash_proto, NULL) + != NULL) + { + continue; + } +} + + nxt_int_t nxt_http_parse_test(nxt_thread_t *thr) { @@ -792,6 +811,8 @@ nxt_http_parse_test(nxt_thread_t *thr) nxt_nitems(nxt_http_test_bench_fields), 0); + nxt_http_parse_test_hash_destroy(&hash); + nxt_memzero(&hash, sizeof(nxt_lvlhsh_t)); lvl_colls = nxt_http_fields_hash_collisions(&hash, @@ -799,6 +820,8 @@ nxt_http_parse_test(nxt_thread_t *thr) nxt_nitems(nxt_http_test_bench_fields), 1); + nxt_http_parse_test_hash_destroy(&hash); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "http parse test hash collisions %ui out of %uz, level: %ui", colls, nxt_nitems(nxt_http_test_bench_fields), lvl_colls); @@ -815,6 +838,7 @@ nxt_http_parse_test(nxt_thread_t *thr) &hash, "simple", 1000000) != NXT_OK) { + nxt_http_parse_test_hash_destroy(&hash); return NXT_ERROR; } @@ -822,9 +846,12 @@ nxt_http_parse_test(nxt_thread_t *thr) &hash, "big", 100000) != NXT_OK) { + nxt_http_parse_test_hash_destroy(&hash); return NXT_ERROR; } + nxt_http_parse_test_hash_destroy(&hash); + return NXT_OK; } From 76fdcce3a1a36f4e8f6b31071a019959ccbc4d2d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 10:45:44 +0000 Subject: [PATCH 237/391] test(port): release the ports and processes the port fixtures build nxt_port_fd_test() and nxt_port_ready_test() (plus its nxt_port_ready_test_stream() helper) build nxt_port_t and nxt_process_t fixtures by hand: nxt_port_new() directly, and a raw nxt_queue_insert_tail() onto the process's ports queue instead of nxt_process_port_add(). Their teardown only called nxt_port_close(), which releases a port's socket and queue mapping but not the port itself -- that needs the reference nxt_port_new() took, dropped with nxt_port_use(task, port, -1). Because the ports were queued without nxt_process_port_add(), port->process was never set, so releasing a port still on that queue would trip the nxt_assert(port->process != NULL) in nxt_port_release(); unlink it from the queue first. The processes were never removed from rt->processes either, which is a malloc'd table outside the test's own pool, so nxt_mp_destroy(mp) never reached it. Under AddressSanitizer with leak detection on this showed up as the rest of the leaks: each port's own memory pool (nxt_port_new() creates one), and the lvlhsh buckets behind rt->processes and rt->ports. Fixes, one per fixture: - nxt_port_ready_test() and nxt_port_ready_test_stream(): unlink each port from its process's queue, drop its reference with nxt_port_use(-1), and remove each process with nxt_runtime_process_remove(). process/queueless/nofd are now initialized to NULL up front like the port variables already were, since the new cleanup reads them on every path through "done", including the ones that jump there before they are assigned. Every removal is gated on process->registered as well as the NULL check: nxt_runtime_process_remove() asserts registered != 0, and both fixtures can reach "done" holding a process that was never added. In the stream helper, proto/router/mainp are allocated up front but registered only partway through -- main_port is created before router is added -- so a main_port failure jumps to "done" with router non-NULL and unregistered. In the outer fixture, nxt_runtime_process_add() itself can fail to allocate its hash node and return with the process unregistered. Without the gate either path aborts the test instead of returning NXT_ERROR. The stream helper's proto_port allocation failure also goes through "done" now instead of returning at once: proto was already in rt->processes, so the early return leaked its hash node. - nxt_port_fd_test(): the second port this test creates is also handed to nxt_port_enable(), which -- correctly -- sets port->pid to this process's own pid before the fragment-handling cases run. That retargets the pid the port was registered under in rt->ports, so nxt_runtime_port_remove() can no longer find it there to release it. Save the original pid and restore it before removing the port. Separately, rt->main_engine was never set on this test's fixture runtime, which crashed nxt_runtime_process_release() with a NULL dereference once teardown actually reached it: it defers a process's free to rt->main_engine unless the calling thread is already on it, and the two must be the same engine for that comparison to hold on a fixture that never leaves the one engine it runs on. The ->registered gate was verified by temporarily forcing the main_port allocation to fail: without it a --debug build aborts with "assertion failed: process->registered != 0"; with it the test returns NXT_ERROR and the rest of the suite continues. There is no hook to make nxt_port_new() fail on demand, so that path has no permanent regression test. Co-Authored-By: Claude Opus 5.5 --- src/test/nxt_port_fd_test.c | 36 ++++++++++++++ src/test/nxt_port_ready_test.c | 88 +++++++++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/src/test/nxt_port_fd_test.c b/src/test/nxt_port_fd_test.c index a16caa14a..13f11d817 100644 --- a/src/test/nxt_port_fd_test.c +++ b/src/test/nxt_port_fd_test.c @@ -364,6 +364,7 @@ nxt_port_fd_test(nxt_thread_t *thr) nxt_fd_t fd0, fd1; nxt_task_t *task; nxt_int_t ret; + nxt_pid_t port_pid; nxt_port_t *port, *existing; nxt_runtime_t *rt, *saved_rt; nxt_port_recv_msg_t msg; @@ -380,6 +381,14 @@ nxt_port_fd_test(nxt_thread_t *thr) port = NULL; existing = NULL; + /* + * Only read at "done" through port, and only ever set together with + * it (see the comment at its real assignment below); the initializer + * is just to satisfy -Wmaybe-uninitialized, which cannot see that + * pairing. + */ + port_pid = 0; + mp = nxt_mp_create(1024, 128, 256, 32); if (nxt_slow_path(mp == NULL)) { return NXT_ERROR; @@ -413,6 +422,15 @@ nxt_port_fd_test(nxt_thread_t *thr) */ thr->engine = engine; + /* + * nxt_runtime_process_release() defers a process's teardown to + * rt->main_engine unless the calling thread is already on it, and + * compares it against thr->engine -- a mismatch posts to whatever + * rt->main_engine happens to hold, NULL here, and crashes. The fixture + * runs everything on the one engine above, so that is main_engine too. + */ + rt->main_engine = engine; + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); /* @@ -579,6 +597,17 @@ nxt_port_fd_test(nxt_thread_t *thr) goto done; } + /* + * nxt_port_enable() below, exercised by the dispatcher cases and the + * fragment tests, sets port->pid to this process's own pid -- the + * usual thing for a port a process enables for itself, but it retargets + * the pid this port was registered in rt->ports under. The teardown + * has to remove it by that original key, since a lookup by the mutated + * pid finds nothing and nxt_runtime_port_remove() would silently leak + * the port and its reference. + */ + port_pid = port->pid; + if (nxt_slow_path(msg.fd[0] != -1 || port->pair[1] != fd0)) { nxt_log_alert(thr->log, "port fd test: the new port did not take the " "socket over"); @@ -709,6 +738,13 @@ nxt_port_fd_test(nxt_thread_t *thr) } if (port != NULL) { + /* + * Restore the pid it was registered under (see the comment at + * port_pid's assignment above) so the lookup inside remove() finds + * it. + */ + port->pid = port_pid; + nxt_port_close(task, port); nxt_runtime_port_remove(task, port); } diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index 2260f5908..3dda9d6d3 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -1009,7 +1009,8 @@ nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, proto_port = nxt_port_new(task, 0, proto->pid, NXT_PROCESS_PROTOTYPE); if (nxt_slow_path(proto_port == NULL)) { - return NXT_ERROR; + /* proto is in rt->processes already: take it out on the way. */ + goto done; } proto_port->pair[0] = -1; @@ -1302,16 +1303,56 @@ nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, done: + /* + * Each port was linked onto its process's queue directly, without + * nxt_process_port_add(), so port->process was never set and + * nxt_port_release() must not be handed a port still on that queue -- + * it would assert. Unlinking first and dropping the reference with + * nxt_port_use() frees the port's own pool (see nxt_port_new()), which + * the caller's nxt_mp_destroy(mp) cannot reach. proto/mainp/router + * need no matching release: they come from mp and are freed along with + * it, but nxt_runtime_process_add() put them in rt->processes, a + * malloc'd table outside mp, and nxt_runtime_process_remove() is what + * takes them back out. + * + * That add happens for each of the three only partway through the + * function -- main_port is created before router is registered -- so + * an early "goto done" can reach here with a process struct that is + * non-NULL but was never added. Removing it anyway would trip + * nxt_runtime_process_remove()'s "process->registered != 0" assert, so + * ->registered gates each removal alongside the NULL check. + */ if (main_port != NULL) { nxt_port_close(task, main_port); + nxt_queue_remove(&main_port->link); + main_port->link.next = NULL; + nxt_port_use(task, main_port, -1); } if (router_port != NULL) { nxt_port_close(task, router_port); + nxt_queue_remove(&router_port->link); + router_port->link.next = NULL; + nxt_port_use(task, router_port, -1); } if (proto_port != NULL) { nxt_port_close(task, proto_port); + nxt_queue_remove(&proto_port->link); + proto_port->link.next = NULL; + nxt_port_use(task, proto_port, -1); + } + + if (mainp != NULL && mainp->registered) { + nxt_runtime_process_remove(rt, mainp); + } + + if (router != NULL && router->registered) { + nxt_runtime_process_remove(rt, router); + } + + if (proto != NULL && proto->registered) { + nxt_runtime_process_remove(rt, proto); } return ret; @@ -1357,6 +1398,11 @@ nxt_port_ready_test(nxt_thread_t *thr) port = NULL; queueless_port = NULL; nofd_port = NULL; + process = NULL; +#if (NXT_HAVE_MEMFD_CREATE) + queueless = NULL; + nofd = NULL; +#endif isolated_pid = nxt_pid + 3; @@ -1643,19 +1689,59 @@ nxt_port_ready_test(nxt_thread_t *thr) * The handler leaves the queue mapping and its descriptor on the port, * and nothing else owns them here: releasing the pool would leak both. * The fixture port never had a socket pair, so this only frees the queue. + * + * Each port was linked onto its process's queue directly, without + * nxt_process_port_add(), so port->process was never set and + * nxt_port_release() must not be handed a port still on that queue -- + * it would assert. Unlinking first and dropping the reference with + * nxt_port_use() frees the port's own pool (see nxt_port_new()), which + * nxt_mp_destroy(mp) below cannot reach. The process itself needs no + * matching release: it came from mp and is freed along with it, but + * nxt_runtime_process_add() put it in rt->processes, a malloc'd table + * outside mp, and nxt_runtime_process_remove() is what takes it back + * out. */ if (port != NULL) { nxt_port_close(task, port); + nxt_queue_remove(&port->link); + port->link.next = NULL; + nxt_port_use(task, port, -1); } if (queueless_port != NULL) { nxt_port_close(task, queueless_port); + nxt_queue_remove(&queueless_port->link); + queueless_port->link.next = NULL; + nxt_port_use(task, queueless_port, -1); } if (nofd_port != NULL) { nxt_port_close(task, nofd_port); + nxt_queue_remove(&nofd_port->link); + nofd_port->link.next = NULL; + nxt_port_use(task, nofd_port, -1); } + /* + * nxt_runtime_process_add() can fail to allocate its hash node and leave + * a process unregistered, so ->registered gates each removal here too. + */ + if (process != NULL && process->registered) { + nxt_runtime_process_remove(rt, process); + } + +#if (NXT_HAVE_MEMFD_CREATE) + + if (queueless != NULL && queueless->registered) { + nxt_runtime_process_remove(rt, queueless); + } + + if (nofd != NULL && nofd->registered) { + nxt_runtime_process_remove(rt, nofd); + } + +#endif + thr->runtime = saved_rt; nxt_thread_mutex_destroy(&rt->processes_mutex); From 90401189e54ec5329d5e63696e6ce599369bb295 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 10:45:59 +0000 Subject: [PATCH 238/391] ci: run the C test binary under ASan with leak detection on The C unit-test suite (build/tests) had no sanitizer coverage in CI: c-tests builds and runs it plain, and sanitize.yml's ASan+UBSan legs build and run unitd itself, with leak detection explicitly off because the daemon forks constantly and that makes LeakSanitizer too noisy to gate on. build/tests does not fork; it is one process running through nxt_lvlhsh, nxt_mp, the port/IPC fixtures and so on directly, so a leak reported there is a real one in that fixture's own code, not daemon-lifecycle noise. Now that the previous two commits fixed every leak LeakSanitizer found in it, add a job that builds it with clang and -fsanitize=address and runs it with LeakSanitizer's default ASAN_OPTIONS (detect_leaks on), so a future leak in this suite fails CI instead of going unnoticed. sudo, same as c-tests, so the clone creds cases still run with real root. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e15e338ca..ac0f67739 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -852,6 +852,47 @@ jobs: - name: Run libunit port_recv fd-ownership test run: ./build/unit_port_recv_test + # Same C unit-test suite as c-tests above, built with AddressSanitizer and + # run with LeakSanitizer's default ASAN_OPTIONS (detect_leaks is on unless + # something turns it off). Separate from sanitize.yml: that workflow runs + # unitd itself, a fork-heavy daemon where leak detection is noisy enough + # that it is turned off there deliberately (see its "Configure unit" + # comment); build/tests is a single non-forking process exercising + # nxt_lvlhsh, nxt_mp, the port/IPC fixtures, etc. directly, so a leak here + # is a real one in that fixture's own teardown, not daemon-lifecycle noise. + # No suppressions file: every known leak was fixed at its allocation site + # (test fixtures freeing what they allocated -- lvlhsh buckets, ports, + # runtime processes). + c-tests-asan: + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v5 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libssl-dev libpcre2-dev clang + + - name: Configure unit (ASan+LSan, --tests) + run: | + ./configure \ + --cc=clang \ + --debug \ + --tests \ + --cc-opt="-fsanitize=address -fno-omit-frame-pointer -g" \ + --ld-opt="-fsanitize=address" + + - name: Build C tests + run: make -j4 tests + + # sudo for the same reason as c-tests above: the clone creds cases + # need real root. Running as root does not disable LeakSanitizer -- + # this repo's own sandboxed reproduction ran as root throughout. + - name: Run C tests + run: sudo ./build/tests + # OpenSSL 4.0 compile gate. 4.0 removed nothing Unit uses, but it deprecated # more of what Unit still calls, and 5.0 is where the deprecated symbols # actually go away -- so the value here is having a leg that fails when a new From 9aaad445bbe32798da27dcac83cfc205cc419d20 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 17 Sep 2026 11:25:40 +0200 Subject: [PATCH 239/391] port: drop a port's pending fd-event changes before it is freed An fd-event change is queued into the engine's batch by a pointer to the event, and the pointer is dereferenced when the batch is committed -- at the top of the next poll at the latest. Nothing kept the struct that pointer names alive until then. port->socket lives inside the port, so nxt_port_release() frees it with port->mem_pool. nxt_port_rearm_now() arms the write event, and nxt_port_write_msgs() calls it immediately before the nxt_port_use() that can take the count to zero and release the port; nothing runs in between. The next nxt_epoll_poll() then writes ev->changing into freed memory and calls epoll_ctl() with a descriptor number the port had closed, which by then may name somebody else's file. A connection does not have this problem: nxt_conn_close_handler() asks nxt_fd_event_close() whether changes are pending and defers the rest of its teardown behind a zero timer, which fires after the next poll. A port has no such handler, never deletes its event from the engine, and is freed from a reference count drop that can happen anywhere. So add the missing engine operation instead: cancel_changes takes an event's pending changes out of the batch. The changes are dropped, not committed -- the descriptor is already closed (nxt_port_mp_cleanup() asserts pair[] is -1), and every facility here removes a descriptor from its set when the descriptor is closed. A caller that wants the kernel told deletes the event first, which is what nxt_fd_event_close() is for. nxt_port_release() calls it, guarded on ->changing so that a port that queued nothing pays nothing. kqueue now sets ->changing in nxt_kqueue_fd_set() so the guard means the same thing there; nothing else in that engine reads the flag. select batches nothing, so its implementation is empty. nxt_fd_event_change_test drives the platform's real engine, not a stub, and asserts on the engine's change count so a leg that stops exercising the path fails instead of passing quietly. Without the call in nxt_port_release() its third leg reports 1 change left and, under -fsanitize=address, a heap-use-after-free in nxt_epoll_commit_changes() on memory freed by nxt_port_release(). Co-Authored-By: Claude Opus 5 (1M context) --- auto/sources | 1 + src/nxt_devpoll_engine.c | 43 +++ src/nxt_epoll_engine.c | 45 +++ src/nxt_event_engine.h | 35 ++ src/nxt_eventport_engine.c | 44 +++ src/nxt_kqueue_engine.c | 70 ++++ src/nxt_poll_engine.c | 43 +++ src/nxt_pollset_engine.c | 43 +++ src/nxt_port.c | 38 ++ src/nxt_select_engine.c | 14 + src/test/nxt_fd_event_change_test.c | 549 ++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 13 files changed, 930 insertions(+) create mode 100644 src/test/nxt_fd_event_change_test.c diff --git a/auto/sources b/auto/sources index 43eb6bbd9..9682b9598 100644 --- a/auto/sources +++ b/auto/sources @@ -168,6 +168,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_http_chunk_parse_test.c \ src/test/nxt_conf_parse_test.c \ src/test/nxt_port_fail_test.c \ + src/test/nxt_fd_event_change_test.c \ src/test/nxt_port_use_unless_zero_test.c \ src/test/nxt_port_mmap_range_test.c \ src/test/nxt_port_ready_test.c \ diff --git a/src/nxt_devpoll_engine.c b/src/nxt_devpoll_engine.c index 71ab11a38..2470d0594 100644 --- a/src/nxt_devpoll_engine.c +++ b/src/nxt_devpoll_engine.c @@ -34,6 +34,8 @@ static void nxt_devpoll_enable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_devpoll_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_devpoll_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_devpoll_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_devpoll_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_devpoll_enable_write(nxt_event_engine_t *engine, @@ -70,6 +72,7 @@ const nxt_event_interface_t nxt_devpoll_engine = { nxt_devpoll_disable, nxt_devpoll_disable, nxt_devpoll_close, + nxt_devpoll_cancel_changes, nxt_devpoll_enable_read, nxt_devpoll_enable_write, nxt_devpoll_disable_read, @@ -390,6 +393,46 @@ nxt_devpoll_change(nxt_event_engine_t *engine, nxt_fd_event_t *ev, } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_devpoll_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, so the change would name a descriptor number that may + * already belong to somebody else. + */ + +static void +nxt_devpoll_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_devpoll_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.devpoll.changes; + end = dst + engine->u.devpoll.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.devpoll.nchanges = (int) (dst - engine->u.devpoll.changes); + + ev->changing = 0; +} + + static nxt_int_t nxt_devpoll_commit_changes(nxt_event_engine_t *engine) { diff --git a/src/nxt_epoll_engine.c b/src/nxt_epoll_engine.c index a4d49ceb0..e1e07b6e5 100644 --- a/src/nxt_epoll_engine.c +++ b/src/nxt_epoll_engine.c @@ -47,6 +47,8 @@ static void nxt_epoll_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_epoll_delete(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_epoll_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_epoll_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_epoll_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_epoll_enable_write(nxt_event_engine_t *engine, @@ -126,6 +128,7 @@ const nxt_event_interface_t nxt_epoll_edge_engine = { nxt_epoll_disable, nxt_epoll_delete, nxt_epoll_close, + nxt_epoll_cancel_changes, nxt_epoll_enable_read, nxt_epoll_enable_write, nxt_epoll_disable_read, @@ -172,6 +175,7 @@ const nxt_event_interface_t nxt_epoll_level_engine = { nxt_epoll_disable, nxt_epoll_delete, nxt_epoll_close, + nxt_epoll_cancel_changes, nxt_epoll_enable_read, nxt_epoll_enable_write, nxt_epoll_disable_read, @@ -412,6 +416,47 @@ nxt_epoll_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev) } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_epoll_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, and close() removes it from the epoll set on its own. + * Committing instead would epoll_ctl() a descriptor number that may already + * name somebody else's file. + */ + +static void +nxt_epoll_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_epoll_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.epoll.changes; + end = dst + engine->u.epoll.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event.data.ptr == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.epoll.nchanges = (nxt_uint_t) (dst - engine->u.epoll.changes); + + ev->changing = 0; +} + + static void nxt_epoll_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev) { diff --git a/src/nxt_event_engine.h b/src/nxt_event_engine.h index 77ed1de3c..4d85786e1 100644 --- a/src/nxt_event_engine.h +++ b/src/nxt_event_engine.h @@ -67,6 +67,37 @@ typedef struct { nxt_bool_t (*close)(nxt_event_engine_t *engine, nxt_fd_event_t *ev); + /* + * Drop every change still pending for this event from the engine's + * change batch. + * + * The batching engines hold the change by pointer and dereference it + * when the batch is committed, at the top of the next poll at the + * latest. Nothing else keeps the struct the pointer names alive until + * then, so anything that frees a struct holding an nxt_fd_event_t has to + * take its pending changes out of the batch first. + * + * The change is dropped, not committed: the descriptor is being closed + * or is closed already, so committing would act on a descriptor number + * that may already name somebody else's file. A caller that wants the + * kernel told must delete the event before it closes, which is what + * nxt_fd_event_close() is for. + * + * For epoll and kqueue that leaves nothing behind: the kernel drops a + * descriptor from its set when the descriptor is closed. The poll, + * devpoll and pollset engines instead keep the set in user space, in + * ->fd_hash and the engine's own array, and only an applied delete + * removes an entry from it. Dropping a pending delete there leaves a + * stale entry pointing at the freed event -- the same entry those + * engines are already left with whenever an event is freed without a + * delete, which is the common case for a port. Not committing is still + * the safer half: committing dereferences the freed event first. + * + * Runs on the engine's own thread, like every other operation here. + */ + void (*cancel_changes)(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); + /* * Add a file descriptor to an event set and enable the most effective * read event notification method provided by underlying event facility. @@ -363,6 +394,10 @@ void nxt_fd_event_hash_destroy(nxt_lvlhsh_t *lvlhsh); (engine)->event.close(engine, ev) +#define nxt_fd_event_cancel_changes(engine, ev) \ + (engine)->event.cancel_changes(engine, ev) + + #define nxt_fd_event_enable_read(engine, ev) \ (engine)->event.enable_read(engine, ev) diff --git a/src/nxt_eventport_engine.c b/src/nxt_eventport_engine.c index 7ac7f44de..a92c96c77 100644 --- a/src/nxt_eventport_engine.c +++ b/src/nxt_eventport_engine.c @@ -23,6 +23,8 @@ static void nxt_eventport_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_eventport_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_eventport_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_eventport_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_eventport_enable_write(nxt_event_engine_t *engine, @@ -63,6 +65,7 @@ const nxt_event_interface_t nxt_eventport_engine = { nxt_eventport_disable, nxt_eventport_disable, nxt_eventport_close, + nxt_eventport_cancel_changes, nxt_eventport_enable_read, nxt_eventport_enable_write, nxt_eventport_disable_read, @@ -293,6 +296,47 @@ nxt_eventport_disable_event(nxt_event_engine_t *engine, nxt_fd_event_t *ev) } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_eventport_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, so the change would name a descriptor number that may + * already belong to somebody else. + */ + +static void +nxt_eventport_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_eventport_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.eventport.changes; + end = dst + engine->u.eventport.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.eventport.nchanges = (nxt_uint_t) + (dst - engine->u.eventport.changes); + + ev->changing = 0; +} + + static nxt_int_t nxt_eventport_commit_changes(nxt_event_engine_t *engine) { diff --git a/src/nxt_kqueue_engine.c b/src/nxt_kqueue_engine.c index e8e282437..822f6e225 100644 --- a/src/nxt_kqueue_engine.c +++ b/src/nxt_kqueue_engine.c @@ -59,6 +59,8 @@ static void nxt_kqueue_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_kqueue_delete(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_kqueue_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_kqueue_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_kqueue_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_kqueue_enable_write(nxt_event_engine_t *engine, @@ -142,6 +144,7 @@ const nxt_event_interface_t nxt_kqueue_engine = { nxt_kqueue_disable, nxt_kqueue_delete, nxt_kqueue_close, + nxt_kqueue_cancel_changes, nxt_kqueue_enable_read, nxt_kqueue_enable_write, nxt_kqueue_disable_read, @@ -313,6 +316,54 @@ nxt_kqueue_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev) } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced when the batch is flushed -- by + * nxt_kqueue_get_kevent() when it fills, or by nxt_kqueue_poll(). + * + * The change is dropped rather than flushed: the descriptor is closed, or is + * about to be, and kqueue(2) removes every kevent that references a + * descriptor when the descriptor is closed. Flushing instead would name a + * descriptor number that may already belong to somebody else. + * + * Matching is on ->udata, which nxt_kqueue_fd_set() sets to the event; that + * names the struct being freed exactly, where ->ident names only a + * descriptor number. nxt_kqueue_file_set() puts a file event in the same + * field, and a pointer comparison tells the two apart on its own. + * + * Unlike the other engines this does not return early when ->changing is + * clear. Here the flag is only ever a "maybe" -- a flush leaves it set, + * for the reason nxt_kqueue_fd_set() gives -- so the scan is what decides, + * and it costs a walk of at most ->mchanges entries. + */ + +static void +nxt_kqueue_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + struct kevent *kev, *dst, *end; + + dst = engine->u.kqueue.changes; + end = dst + engine->u.kqueue.nchanges; + + for (kev = dst; kev < end; kev++) { + + if (nxt_kevent_get_udata(kev->udata) == (void *) ev) { + continue; + } + + if (dst != kev) { + *dst = *kev; + } + + dst++; + } + + engine->u.kqueue.nchanges = (int) (dst - engine->u.kqueue.changes); + + ev->changing = 0; +} + + /* * The kqueue event engine uses only three states: inactive, blocked, and * active. An active oneshot event is marked as it is in the default @@ -451,6 +502,25 @@ nxt_kqueue_fd_set(nxt_event_engine_t *engine, nxt_fd_event_t *ev, kev = nxt_kqueue_get_kevent(engine); + /* + * ->changing says a change for this event may be buffered. It is what + * the generic callers test before they ask for a cancel, so kqueue has + * to set it even though no kqueue path reads it -- nxt_kqueue_close() + * scans by descriptor, and nxt_kqueue_cancel_changes() scans by ->udata + * without testing the flag. + * + * A flush does not clear it, because the batch mixes fd events with the + * file events nxt_kqueue_file_set() puts in the same ->udata field, and + * nothing in a kevent says which of the two it holds: clearing the flag + * over a flushed batch would write through an nxt_file_event_t as if it + * were an nxt_fd_event_t. So the flag only ever says "maybe", and the + * cost of that is a scan that finds nothing. It is never stale in the + * unsafe direction: it is set whenever a change is queued, and cleared + * only once the batch has been scanned. + */ + + ev->changing = 1; + kev->ident = ev->fd; kev->filter = filter; kev->flags = flags; diff --git a/src/nxt_poll_engine.c b/src/nxt_poll_engine.c index f514e0a9a..63d126c33 100644 --- a/src/nxt_poll_engine.c +++ b/src/nxt_poll_engine.c @@ -34,6 +34,8 @@ static void nxt_poll_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_poll_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_poll_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_poll_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_poll_enable_write(nxt_event_engine_t *engine, @@ -73,6 +75,7 @@ const nxt_event_interface_t nxt_poll_engine = { nxt_poll_disable, nxt_poll_disable, nxt_poll_close, + nxt_poll_cancel_changes, nxt_poll_enable_read, nxt_poll_enable_write, nxt_poll_disable_read, @@ -318,6 +321,46 @@ nxt_poll_change(nxt_event_engine_t *engine, nxt_fd_event_t *ev, nxt_uint_t op, } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_poll_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, so the change would name a descriptor number that may + * already belong to somebody else. + */ + +static void +nxt_poll_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_poll_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.poll.changes; + end = dst + engine->u.poll.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.poll.nchanges = (nxt_uint_t) (dst - engine->u.poll.changes); + + ev->changing = 0; +} + + static nxt_int_t nxt_poll_commit_changes(nxt_event_engine_t *engine) { diff --git a/src/nxt_pollset_engine.c b/src/nxt_pollset_engine.c index e2c1a2d01..1d439b457 100644 --- a/src/nxt_pollset_engine.c +++ b/src/nxt_pollset_engine.c @@ -29,6 +29,8 @@ static void nxt_pollset_enable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_pollset_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_pollset_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_pollset_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_pollset_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_pollset_enable_write(nxt_event_engine_t *engine, @@ -64,6 +66,7 @@ const nxt_event_interface_t nxt_pollset_engine = { nxt_pollset_disable, nxt_pollset_disable, nxt_pollset_close, + nxt_pollset_cancel_changes, nxt_pollset_enable_read, nxt_pollset_enable_write, nxt_pollset_disable_read, @@ -376,6 +379,46 @@ nxt_pollset_change(nxt_event_engine_t *engine, nxt_fd_event_t *ev, } +/* + * Take this event's pending changes out of the batch, so that a struct that + * is about to be freed is not dereferenced by nxt_pollset_commit_changes(). + * + * The change is dropped rather than committed: the descriptor is closed, or + * is about to be, so the change would name a descriptor number that may + * already belong to somebody else. + */ + +static void +nxt_pollset_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ + nxt_pollset_change_t *change, *dst, *end; + + if (!ev->changing) { + return; + } + + dst = engine->u.pollset.changes; + end = dst + engine->u.pollset.nchanges; + + for (change = dst; change < end; change++) { + + if (change->event == ev) { + continue; + } + + if (dst != change) { + *dst = *change; + } + + dst++; + } + + engine->u.pollset.nchanges = (int) (dst - engine->u.pollset.changes); + + ev->changing = 0; +} + + static nxt_int_t nxt_pollset_commit_changes(nxt_event_engine_t *engine) { diff --git a/src/nxt_port.c b/src/nxt_port.c index 1ae594628..b05e1b598 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -235,6 +235,44 @@ nxt_port_release(nxt_task_t *task, nxt_port_t *port) port->app = NULL; + /* + * port->socket is embedded in the port, so it dies with the memory pool + * released at the end of this function. A change queued for it is held + * by pointer in the engine's change batch and is dereferenced when that + * batch is committed, which happens at the top of the next poll at the + * latest -- after this release. nxt_port_rearm_now() queues exactly such + * a change, and nxt_port_write_msgs() calls it immediately before the + * nxt_port_use() that can bring the count to zero and reach here. + * + * Unlike a connection, a port has no close handler to defer its own + * teardown behind the next poll (nxt_conn_close_handler() does that with + * a zero timer), and it never deletes its event from the engine. So the + * pending change is dropped instead. + * + * A port with an engine is released on that engine's thread: the only + * other path through nxt_port_use() carries the last drop across with a + * work item. A port that never reached nxt_port_read_enable() or + * nxt_port_write_enable() has no engine and has queued nothing. + * + * ->socket.changing is the engines' own "a change may be buffered" bit. + * On kqueue it is only ever a maybe -- set when a change is queued and + * cleared only by the cancel, never by a flush -- so the test here can + * ask for a scan that finds nothing, but it never skips a cancel that + * was needed. + * + * port->engine is read as the engine the change was queued on, which it + * is everywhere but one path: nxt_router_thread_exit_handler() points a + * departed worker's port at the router's own engine before it drops the + * last reference. A change left in the departed engine's batch is not + * found by the scan below, and is not committed either -- that engine is + * never polled again and nxt_event_engine_free() releases it, batch and + * all, a few lines further on. + */ + + if (port->engine != NULL && port->socket.changing) { + nxt_fd_event_cancel_changes(port->engine, &port->socket); + } + if (port->link.next != NULL) { nxt_assert(port->process != NULL); diff --git a/src/nxt_select_engine.c b/src/nxt_select_engine.c index 6f7600125..8f268a802 100644 --- a/src/nxt_select_engine.c +++ b/src/nxt_select_engine.c @@ -14,6 +14,8 @@ static void nxt_select_enable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_select_disable(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static nxt_bool_t nxt_select_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev); +static void nxt_select_cancel_changes(nxt_event_engine_t *engine, + nxt_fd_event_t *ev); static void nxt_select_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev); static void nxt_select_enable_write(nxt_event_engine_t *engine, @@ -42,6 +44,7 @@ const nxt_event_interface_t nxt_select_engine = { nxt_select_disable, nxt_select_disable, nxt_select_close, + nxt_select_cancel_changes, nxt_select_enable_read, nxt_select_enable_write, nxt_select_disable_read, @@ -124,6 +127,17 @@ nxt_select_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev) } +/* + * select() applies every change as it is made and batches nothing, so this + * engine never sets ->changing and holds no pending change to drop. + */ + +static void +nxt_select_cancel_changes(nxt_event_engine_t *engine, nxt_fd_event_t *ev) +{ +} + + static void nxt_select_enable_read(nxt_event_engine_t *engine, nxt_fd_event_t *ev) { diff --git a/src/test/nxt_fd_event_change_test.c b/src/test/nxt_fd_event_change_test.c new file mode 100644 index 000000000..f95e3b1c6 --- /dev/null +++ b/src/test/nxt_fd_event_change_test.c @@ -0,0 +1,549 @@ +/* + * Copyright (C) NGINX, Inc. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +/* + * A queued fd-event change is held by pointer in the engine's change batch + * and dereferenced when that batch is committed, at the top of the next poll + * at the latest. Nothing else keeps the struct the pointer names alive, so + * anything that frees such a struct has to drop its pending changes first -- + * nxt_fd_event_cancel_changes(). + * + * These legs run against the platform's real event engine, not a stub: the + * batch and its commit are the thing under test. Each asserts on the + * engine's change count, so a leg that stops doing what it claims fails + * rather than passing quietly. Under a sanitized build the final poll in + * each leg is the second oracle: it commits whatever is left, so a change + * that was not dropped is reported as a use-after-free. + */ + +/* + * ->changing is not a cross-engine contract. The epoll engine keeps it and + * clears it when it commits, so the legs below can assert on it there. The + * kqueue engine does not keep it at all -- its paths work by descriptor or + * by ->udata -- so on kqueue the change count is the only oracle. + */ + +#if (NXT_HAVE_EPOLL) +#define nxt_fd_event_change_test_nchanges(engine) \ + ((nxt_uint_t) (engine)->u.epoll.nchanges) +#define NXT_FD_EVENT_CHANGE_TEST 1 +#define NXT_FD_EVENT_CHANGE_TEST_CHANGING 1 + +#elif (NXT_HAVE_KQUEUE) +#define nxt_fd_event_change_test_nchanges(engine) \ + ((nxt_uint_t) (engine)->u.kqueue.nchanges) +#define NXT_FD_EVENT_CHANGE_TEST 1 +#define NXT_FD_EVENT_CHANGE_TEST_CHANGING 0 + +#else +#define NXT_FD_EVENT_CHANGE_TEST 0 +#define NXT_FD_EVENT_CHANGE_TEST_CHANGING 0 +#endif + + +#if (NXT_FD_EVENT_CHANGE_TEST) + +static nxt_event_engine_t *nxt_fd_event_change_test_engine(nxt_task_t *task); +static nxt_int_t nxt_fd_event_change_test_drop(nxt_thread_t *thr, + nxt_event_engine_t *engine); +static nxt_int_t nxt_fd_event_change_test_keeps_others(nxt_thread_t *thr, + nxt_event_engine_t *engine); +static nxt_int_t nxt_fd_event_change_test_port(nxt_thread_t *thr, + nxt_event_engine_t *engine); +static nxt_fd_event_t *nxt_fd_event_change_test_event(nxt_task_t *task, + nxt_fd_t fd); +static void nxt_fd_event_change_test_handler(nxt_task_t *task, void *obj, + void *data); + + +nxt_int_t +nxt_fd_event_change_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_event_engine_t *engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "fd event change test started"); + + task = thr->task; + task->thread = thr; + + engine = nxt_fd_event_change_test_engine(task); + if (nxt_slow_path(engine == NULL)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: no event engine"); + return NXT_ERROR; + } + + saved_engine = thr->engine; + thr->engine = engine; + + ret = nxt_fd_event_change_test_drop(thr, engine); + + if (ret == NXT_OK) { + ret = nxt_fd_event_change_test_keeps_others(thr, engine); + } + + if (ret == NXT_OK) { + ret = nxt_fd_event_change_test_port(thr, engine); + } + + thr->engine = saved_engine; + + nxt_event_engine_free(engine); + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "fd event change test passed"); + + return NXT_OK; +} + + +static nxt_event_engine_t * +nxt_fd_event_change_test_engine(nxt_task_t *task) +{ + nxt_mp_t *mp; + nxt_array_t *services; + nxt_event_engine_t *engine; + const nxt_event_interface_t *interface; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NULL; + } + + services = nxt_services_init(mp); + if (nxt_slow_path(services == NULL)) { + nxt_mp_destroy(mp); + return NULL; + } + + interface = nxt_service_get(services, "engine", NULL); + if (nxt_slow_path(interface == NULL)) { + nxt_mp_destroy(mp); + return NULL; + } + + /* + * No signal set: this fixture drives the engine by hand and never runs + * nxt_event_engine_start(), so nothing here delivers a signal. + */ + + engine = nxt_event_engine_create(task, interface, NULL, 0, 0); + + nxt_mp_destroy(mp); + + return engine; +} + + +static nxt_fd_event_t * +nxt_fd_event_change_test_event(nxt_task_t *task, nxt_fd_t fd) +{ + nxt_fd_event_t *ev; + + /* + * nxt_malloc(), not the mem pool a port uses, so that the poll at the + * end of the leg reads freed memory outright under a sanitizer rather + * than memory a pool happens to keep mapped. + */ + + ev = nxt_zalloc(sizeof(nxt_fd_event_t)); + if (nxt_slow_path(ev == NULL)) { + return NULL; + } + + ev->fd = fd; + ev->task = task; + ev->log = task->log; + ev->read_work_queue = &task->thread->engine->fast_work_queue; + ev->write_work_queue = &task->thread->engine->fast_work_queue; + ev->read_handler = nxt_fd_event_change_test_handler; + ev->write_handler = nxt_fd_event_change_test_handler; + ev->error_handler = nxt_fd_event_change_test_handler; + + return ev; +} + + +static void +nxt_fd_event_change_test_handler(nxt_task_t *task, void *obj, void *data) +{ +} + + +/* + * The bare mechanism: a change queued for an event, then dropped, leaves the + * batch empty, so freeing the event afterwards cannot be reached by a commit. + */ + +static nxt_int_t +nxt_fd_event_change_test_drop(nxt_thread_t *thr, nxt_event_engine_t *engine) +{ + nxt_int_t ret; + nxt_fd_event_t *ev; + nxt_socket_t pair[2]; + + ret = NXT_ERROR; + ev = NULL; + pair[0] = -1; + pair[1] = -1; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: socketpair failed"); + return NXT_ERROR; + } + + ev = nxt_fd_event_change_test_event(thr->task, pair[1]); + if (nxt_slow_path(ev == NULL)) { + goto done; + } + + nxt_fd_event_enable_write(engine, ev); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: enabling the write event queued " + "%ui changes, expected 1", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + +#if (NXT_FD_EVENT_CHANGE_TEST_CHANGING) + if (nxt_slow_path(ev->changing != 1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: a queued change left changing:%d", + (int) ev->changing); + goto done; + } +#endif + + nxt_fd_event_cancel_changes(engine, ev); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: cancelling left %ui changes", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + +#if (NXT_FD_EVENT_CHANGE_TEST_CHANGING) + if (nxt_slow_path(ev->changing != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: cancelling left changing:%d", + (int) ev->changing); + goto done; + } +#endif + + nxt_free(ev); + ev = NULL; + + /* Commits whatever is left. Nothing is, so nothing reads the free. */ + + engine->event.poll(engine, 0); + + ret = NXT_OK; + +done: + + nxt_free(ev); + + if (pair[0] != -1) { + nxt_socket_close(thr->task, pair[0]); + } + + if (pair[1] != -1) { + nxt_socket_close(thr->task, pair[1]); + } + + return ret; +} + + +/* + * Cancelling one event's change leaves every other event's change in the + * batch, and still commits it. The compaction is the part of this that can + * silently go wrong. + */ + +static nxt_int_t +nxt_fd_event_change_test_keeps_others(nxt_thread_t *thr, + nxt_event_engine_t *engine) +{ + nxt_int_t ret; + nxt_uint_t i; + nxt_fd_event_t *ev[3]; + nxt_socket_t pair[3][2]; + + ret = NXT_ERROR; + + for (i = 0; i < 3; i++) { + ev[i] = NULL; + pair[i][0] = -1; + pair[i][1] = -1; + } + + for (i = 0; i < 3; i++) { + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair[i]) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: socketpair failed"); + goto done; + } + + ev[i] = nxt_fd_event_change_test_event(thr->task, pair[i][1]); + if (nxt_slow_path(ev[i] == NULL)) { + goto done; + } + + nxt_fd_event_enable_write(engine, ev[i]); + } + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 3)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: three events queued %ui changes", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + + /* The middle one, so that compaction has to move a tail entry. */ + + nxt_fd_event_cancel_changes(engine, ev[1]); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 2)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: cancelling one of three left %ui " + "changes, expected 2", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + +#if (NXT_FD_EVENT_CHANGE_TEST_CHANGING) + if (nxt_slow_path(ev[0]->changing != 1 || ev[2]->changing != 1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: cancelling one event cleared " + "changing on another (%d, %d)", + (int) ev[0]->changing, (int) ev[2]->changing); + goto done; + } +#endif + + nxt_free(ev[1]); + ev[1] = NULL; + + /* + * Commits the two that were kept. Both are still allocated, and both + * descriptors are still open, so this is the check that compaction left + * usable entries rather than merely the right count. + */ + + engine->event.poll(engine, 0); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: the poll left %ui changes", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + +#if (NXT_FD_EVENT_CHANGE_TEST_CHANGING) + if (nxt_slow_path(ev[0]->changing != 0 || ev[2]->changing != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: a committed change left " + "changing set (%d, %d)", + (int) ev[0]->changing, (int) ev[2]->changing); + goto done; + } +#endif + +#if (NXT_HAVE_EPOLL) + /* + * The count above says two entries survived; this says they are the + * right two. A swapped entry commits silently and keeps the count, so + * ask the kernel instead: the kept descriptors are in the epoll set and + * EPOLL_CTL_MOD finds them, the cancelled one was never added and + * EPOLL_CTL_MOD reports ENOENT. + */ + + { + int err; + nxt_uint_t k; + struct epoll_event ee; + + for (k = 0; k < 3; k++) { + nxt_memzero(&ee, sizeof(struct epoll_event)); + ee.events = EPOLLOUT; + ee.data.ptr = ev[k]; + + err = epoll_ctl(engine->u.epoll.fd, EPOLL_CTL_MOD, pair[k][1], + &ee); + + if (k == 1) { + if (nxt_slow_path(err == 0 || nxt_errno != NXT_ENOENT)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: the cancelled event " + "reached the epoll set (fd:%d err:%d)", + pair[k][1], err); + goto done; + } + + } else if (nxt_slow_path(err != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: a kept event is not in " + "the epoll set (fd:%d) %E", + pair[k][1], nxt_errno); + goto done; + } + } + } +#endif + + ret = NXT_OK; + +done: + + for (i = 0; i < 3; i++) { + nxt_free(ev[i]); + + if (pair[i][0] != -1) { + nxt_socket_close(thr->task, pair[i][0]); + } + + if (pair[i][1] != -1) { + nxt_socket_close(thr->task, pair[i][1]); + } + } + + return ret; +} + + +/* + * The reachable case. nxt_port_rearm_now() arms the write event, and + * nxt_port_write_msgs() calls it immediately before the nxt_port_use() that + * can take the count to zero -- which releases the port's memory pool, and + * port->socket lives in it. This leg is that pair of calls with nothing in + * between: arm, then drop the last reference. + * + * Before the fix the poll below commits a change into freed memory. Under a + * sanitized build that is a heap-use-after-free report; in a plain build it + * is silent, which is why the change count is asserted as well. + */ + +static nxt_int_t +nxt_fd_event_change_test_port(nxt_thread_t *thr, nxt_event_engine_t *engine) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + nxt_socket_t pair[2]; + + ret = NXT_ERROR; + task = thr->task; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: socketpair failed"); + return NXT_ERROR; + } + + port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(port == NULL)) { + nxt_socket_close(task, pair[0]); + nxt_socket_close(task, pair[1]); + return NXT_ERROR; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.task = task; + + nxt_port_write_enable(task, port); + + if (nxt_slow_path(port->engine != engine)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: the port took another engine"); + goto done; + } + + /* Same engine, so this arms the write event inline. */ + + nxt_port_rearm(task, port); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 1)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: re-arming the port queued %ui " + "changes, expected 1", + nxt_fd_event_change_test_nchanges(engine)); + goto done; + } + + ret = NXT_OK; + +done: + + /* + * What nxt_port_close() does to the descriptors, which every release + * path runs first -- nxt_port_mp_cleanup() asserts on it. port->socket + * keeps the descriptor number it had, so a change left in the batch now + * names a number the next open() may hand to somebody else. + */ + + nxt_socket_close(task, port->pair[0]); + port->pair[0] = -1; + + nxt_socket_close(task, port->pair[1]); + port->pair[1] = -1; + + /* + * The last reference. nxt_port_release() runs from here and releases + * port->mem_pool, which is where port->socket lives. + */ + + nxt_port_use(task, port, -1); + + if (ret != NXT_OK) { + return ret; + } + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: releasing the port left %ui " + "changes pointing into its freed memory pool", + nxt_fd_event_change_test_nchanges(engine)); + return NXT_ERROR; + } + + /* Would commit that change, if one were left. */ + + engine->event.poll(engine, 0); + + return NXT_OK; +} + +#else /* !NXT_FD_EVENT_CHANGE_TEST */ + +nxt_int_t +nxt_fd_event_change_test(nxt_thread_t *thr) +{ + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test skipped: the change batch is asserted " + "on only for epoll and kqueue"); + + return NXT_OK; +} + +#endif diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index b2411af70..da6d7d84d 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -195,6 +195,10 @@ main(int argc, char **argv) return 1; } + if (nxt_fd_event_change_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_use_unless_zero_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 8c596cd72..c6dcafeec 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -70,6 +70,7 @@ nxt_int_t nxt_http_chunk_parse_test(nxt_thread_t *thr); nxt_int_t nxt_conf_json_depth_test(nxt_thread_t *thr); nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); +nxt_int_t nxt_fd_event_change_test(nxt_thread_t *thr); nxt_int_t nxt_port_use_unless_zero_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); From 356fc2583186c911d3bcefdfc68ad849b6d38c68 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 17 Sep 2026 11:25:40 +0200 Subject: [PATCH 240/391] changelog: a freed port no longer leaves a pending fd-event change Co-Authored-By: Claude Opus 5 (1M context) --- CHANGES | 9 +++++++++ docs/changes.xml | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGES b/CHANGES index 182befd2c..ed2f860c2 100644 --- a/CHANGES +++ b/CHANGES @@ -60,6 +60,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 is not valid UTF-8 with U+FFFD before it serializes. *) Change: upgrade contrib njs to 1.0.1. + + *) Bugfix: a port that was closed and freed could leave a pending epoll + or kqueue change behind it. The change is held by a pointer into the + port, and is applied when the batch is committed -- at the top of the + next poll at the latest, which is after the port's memory pool is + released. The commit then read freed memory, and could also name a + descriptor number the kernel had already handed to somebody else. A + port now drops its pending changes before it is freed (#414). + *) Bugfix: a process could stop hearing from a peer for good. When the wake-up that tells a peer to read the shared queue failed because the kernel could not allocate for it, the port treated the peer as diff --git a/docs/changes.xml b/docs/changes.xml index 5d23698ed..9313b7489 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -103,6 +103,18 @@ never filled up will not deliver (#392). + + +a port that was closed and freed could leave a pending epoll or kqueue change +behind it. The change is held by a pointer into the port, and is applied when +the batch is committed -- at the top of the next poll at the latest, which is +after the port's memory pool is released. The commit then read freed memory, +and could also name a descriptor number the kernel had already handed to +somebody else. A port now drops its pending changes before it is freed +(#414). + + + upgrade contrib njs to 1.0.1. From 0c4199b1704981a193636162b7bada78bceda369 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 18:20:23 +0000 Subject: [PATCH 241/391] test: commit the engine's own changes before counting a leg's Without eventfd (epoll) or EVFILT_USER (kqueue) the engine posts through a signal pipe, and nxt_event_engine_create() enables that pipe's read event in the same change batch the legs then count. The first leg saw 2 changes where it queued 1, and the suite failed on those builds: fd event change test: enabling the write event queued 2 changes, expected 1 Poll the fixture engine once before the legs, and refuse to run them if the batch is not empty afterwards, so each leg counts only what it queued. Reproduced by undefining NXT_HAVE_EVENTFD in the generated config; passes there and in the default build. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FXFwqBMUbo35vhzidxWQpp --- src/test/nxt_fd_event_change_test.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/test/nxt_fd_event_change_test.c b/src/test/nxt_fd_event_change_test.c index f95e3b1c6..5dd50ddca 100644 --- a/src/test/nxt_fd_event_change_test.c +++ b/src/test/nxt_fd_event_change_test.c @@ -86,6 +86,24 @@ nxt_fd_event_change_test(nxt_thread_t *thr) saved_engine = thr->engine; thr->engine = engine; + /* + * Without eventfd (epoll) or EVFILT_USER (kqueue) the engine posts + * through a signal pipe, and nxt_event_engine_create() enables its read + * event in this same batch. Commit it, so that every leg counts only + * the changes it queued itself. + */ + + engine->event.poll(engine, 0); + + if (nxt_slow_path(nxt_fd_event_change_test_nchanges(engine) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "fd event change test: the engine's own changes were " + "not committed, %ui left", + nxt_fd_event_change_test_nchanges(engine)); + ret = NXT_ERROR; + goto done; + } + ret = nxt_fd_event_change_test_drop(thr, engine); if (ret == NXT_OK) { @@ -96,6 +114,8 @@ nxt_fd_event_change_test(nxt_thread_t *thr) ret = nxt_fd_event_change_test_port(thr, engine); } +done: + thr->engine = saved_engine; nxt_event_engine_free(engine); From 530ccddb7cd6749c8782b699fb7e4a6f5b9da346 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 18:28:07 +0000 Subject: [PATCH 242/391] kqueue: give the real reason a flush leaves ->changing set The comment said nothing in a kevent tells an fd event from a file event, so clearing the flag over a flushed batch would write through the wrong struct. ->filter does tell them apart, and nxt_kqueue_error() already relies on it. The flag is left set because clearing it would walk the batch on every flush and every poll to spare a scan that runs only when a port is released. Both comments also named nxt_kqueue_file_set(), which does not exist; the function is nxt_kqueue_enable_file(). In the test, log the errno epoll_ctl() left rather than its return value, read before anything can overwrite it, and say that leg 2's poll queues handlers holding the kept events, which nothing runs before the engine is freed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FXFwqBMUbo35vhzidxWQpp --- src/nxt_kqueue_engine.c | 20 ++++++++++---------- src/test/nxt_fd_event_change_test.c | 15 +++++++++++---- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/nxt_kqueue_engine.c b/src/nxt_kqueue_engine.c index 822f6e225..d3b8c3be6 100644 --- a/src/nxt_kqueue_engine.c +++ b/src/nxt_kqueue_engine.c @@ -328,8 +328,8 @@ nxt_kqueue_close(nxt_event_engine_t *engine, nxt_fd_event_t *ev) * * Matching is on ->udata, which nxt_kqueue_fd_set() sets to the event; that * names the struct being freed exactly, where ->ident names only a - * descriptor number. nxt_kqueue_file_set() puts a file event in the same - * field, and a pointer comparison tells the two apart on its own. + * descriptor number. nxt_kqueue_enable_file() puts a file event in the + * same field, and a pointer comparison tells the two apart on its own. * * Unlike the other engines this does not return early when ->changing is * clear. Here the flag is only ever a "maybe" -- a flush leaves it set, @@ -509,14 +509,14 @@ nxt_kqueue_fd_set(nxt_event_engine_t *engine, nxt_fd_event_t *ev, * scans by descriptor, and nxt_kqueue_cancel_changes() scans by ->udata * without testing the flag. * - * A flush does not clear it, because the batch mixes fd events with the - * file events nxt_kqueue_file_set() puts in the same ->udata field, and - * nothing in a kevent says which of the two it holds: clearing the flag - * over a flushed batch would write through an nxt_file_event_t as if it - * were an nxt_fd_event_t. So the flag only ever says "maybe", and the - * cost of that is a scan that finds nothing. It is never stale in the - * unsafe direction: it is set whenever a change is queued, and cleared - * only once the batch has been scanned. + * A flush does not clear it. It could: ->filter tells an fd event from + * the file events nxt_kqueue_enable_file() puts in the same ->udata + * field, as nxt_kqueue_error() relies on. But clearing would walk the + * whole batch on every flush and every poll, to spare a scan that runs + * only when a port is released. So the flag only ever says "maybe", and + * the cost of that is a scan that finds nothing. It is never stale in + * the unsafe direction: it is set whenever a change is queued, and + * cleared only once the batch has been scanned. */ ev->changing = 1; diff --git a/src/test/nxt_fd_event_change_test.c b/src/test/nxt_fd_event_change_test.c index 5dd50ddca..89417c760 100644 --- a/src/test/nxt_fd_event_change_test.c +++ b/src/test/nxt_fd_event_change_test.c @@ -368,6 +368,11 @@ nxt_fd_event_change_test_keeps_others(nxt_thread_t *thr, * Commits the two that were kept. Both are still allocated, and both * descriptors are still open, so this is the check that compaction left * usable entries rather than merely the right count. + * + * Both are writable, so the poll also queues their handlers on + * fast_work_queue with ev[0] and ev[2] as the object. Nothing runs that + * queue before nxt_event_engine_free() drops it; a leg that does run it + * has to do so before the events are freed below. */ engine->event.poll(engine, 0); @@ -400,6 +405,7 @@ nxt_fd_event_change_test_keeps_others(nxt_thread_t *thr, { int err; + nxt_err_t errn; nxt_uint_t k; struct epoll_event ee; @@ -410,13 +416,14 @@ nxt_fd_event_change_test_keeps_others(nxt_thread_t *thr, err = epoll_ctl(engine->u.epoll.fd, EPOLL_CTL_MOD, pair[k][1], &ee); + errn = (err == 0) ? 0 : nxt_errno; if (k == 1) { - if (nxt_slow_path(err == 0 || nxt_errno != NXT_ENOENT)) { + if (nxt_slow_path(err == 0 || errn != NXT_ENOENT)) { nxt_log_error(NXT_LOG_NOTICE, thr->log, "fd event change test: the cancelled event " - "reached the epoll set (fd:%d err:%d)", - pair[k][1], err); + "reached the epoll set (fd:%d) %E", + pair[k][1], errn); goto done; } @@ -424,7 +431,7 @@ nxt_fd_event_change_test_keeps_others(nxt_thread_t *thr, nxt_log_error(NXT_LOG_NOTICE, thr->log, "fd event change test: a kept event is not in " "the epoll set (fd:%d) %E", - pair[k][1], nxt_errno); + pair[k][1], errn); goto done; } } From 8a24a9f06a2233eab0aa99c25b9b133de0783012 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 25 Sep 2026 20:37:49 +0200 Subject: [PATCH 243/391] test: step control messages with NXT_CMSG_NXTHDR On musl with clang, "make tests" fails with -Werror=sign-compare in nxt_port_ctrunc_test.c and nxt_port_queued_fd_test.c. musl's CMSG_NXTHDR macro compares a size_t against a ptrdiff_t; glibc is unaffected because its CMSG_NXTHDR is an inline function. The "all" target already builds, so no CI leg sees this. nxt_socket_msg.h wraps the macro in NXT_CMSG_NXTHDR with the warning suppressed under !__GLIBC__ && __clang__, which is what production code uses. Switch both test loops to it. CMSG_FIRSTHDR casts to size_t on musl and needs no change. Co-Authored-By: Claude Fable 5.1 --- src/test/nxt_port_ctrunc_test.c | 2 +- src/test/nxt_port_queued_fd_test.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/nxt_port_ctrunc_test.c b/src/test/nxt_port_ctrunc_test.c index 3e489d843..5b6fec913 100644 --- a/src/test/nxt_port_ctrunc_test.c +++ b/src/test/nxt_port_ctrunc_test.c @@ -288,7 +288,7 @@ nxt_port_ctrunc_test_probe(size_t controllen, size_t *used, int *flags) for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; - cmsg = CMSG_NXTHDR(&msg, cmsg)) + cmsg = NXT_CMSG_NXTHDR(&msg, cmsg)) { if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) diff --git a/src/test/nxt_port_queued_fd_test.c b/src/test/nxt_port_queued_fd_test.c index a99d63154..89db817d7 100644 --- a/src/test/nxt_port_queued_fd_test.c +++ b/src/test/nxt_port_queued_fd_test.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "nxt_tests.h" #include @@ -568,7 +569,7 @@ nxt_port_queued_fd_test_recv_fd(nxt_fd_t sock) fd = -1; - for (cm = CMSG_FIRSTHDR(&mh); cm != NULL; cm = CMSG_NXTHDR(&mh, cm)) { + for (cm = CMSG_FIRSTHDR(&mh); cm != NULL; cm = NXT_CMSG_NXTHDR(&mh, cm)) { if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SCM_RIGHTS) { nxt_memcpy(&fd, CMSG_DATA(cm), sizeof(int)); break; From 43e5cf1385dda3388383d4e4850dbfbe1e0e64e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 19:00:07 +0000 Subject: [PATCH 244/391] ci(distros): build the tests binary on the rolling legs Both rolling legs configure with --tests but built only "all", so a toolchain change that breaks the tests binary alone was invisible: on Alpine edge, musl's CMSG_NXTHDR() tripped -Wsign-compare under clang in two test loops while "make" stayed green (#475). Build "tests" right after "all", before the module configures rewrite the build. That also puts musl + gcc on those files for the first time, since the wrapper #475 relies on silences the warning only under clang. Built, not run: the clone and credential cases need capabilities the container may not grant, and c-tests already runs the binary on the host runner. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FXFwqBMUbo35vhzidxWQpp --- .github/workflows/build-distros.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index e75ce5794..5d29c0b88 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -56,6 +56,13 @@ jobs: - name: make unit run: make -j 4 + # --tests has always been configured here, but only "all" was built, + # so a toolchain that breaks the tests binary alone went unnoticed + # (musl's CMSG_NXTHDR under clang, #475). Built, not run: the clone + # and credential cases need capabilities this container may not have. + - name: make tests + run: make -j 4 tests + - name: configure unit-php run: ./configure php @@ -176,6 +183,13 @@ jobs: - name: make unit run: make -j 4 + # --tests has always been configured here, but only "all" was built, + # so a toolchain that breaks the tests binary alone went unnoticed + # (musl's CMSG_NXTHDR under clang, #475). Built, not run: the clone + # and credential cases need capabilities this container may not have. + - name: make tests + run: make -j 4 tests + - name: configure unit-php run: | ln -s /usr/lib/libphp83.so /usr/lib/libphp.so From 50b58bf3a701e1a594fb7206672e4cb88c0f9b8f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 25 Sep 2026 20:32:43 +0200 Subject: [PATCH 245/391] ci: drop the dead CLANG_PATH pin from the wasm-wasi-component leg The "Make wasm-wasi-component" step set CLANG_PATH=/usr/bin/clang-15. The runner is ubuntu-latest, which has been Ubuntu 24.04 since early 2025 and ships clang 16, 17 and 18 only, so the path does not exist. clang-sys (1.8.1 in src/wasm-wasi-component/Cargo.lock) treats a CLANG_PATH that is not an executable file as a warning on stderr and falls through to a PATH search; cargo hides that warning on success. The leg has therefore been building bindgen against the runner's clang 18 all along, and the green master run 35533290038 confirms it. The pin dates from 0cee7d1a (2024-03-06), when ubuntu-latest was 22.04 and did ship clang-15. It is not replaced with a newer pin: that would rot the same way. The real hazard, bindgen against a new libclang, is what build-distros.yml checks on Fedora rawhide and Alpine edge. No CHANGES entry: CI-only, nothing user-visible changes. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/build-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ac0f67739..d4e715f04 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -702,9 +702,10 @@ jobs: ./configure wasm-wasi-component if: steps.metadata.outputs.module == 'wasm-wasi-component' + # bindgen picks clang up from PATH. The libclang-drift check against + # new distribution compilers lives in build-distros.yml. - name: Make wasm-wasi-component run: | - CLANG_PATH=/usr/bin/clang-15 \ BINDGEN_EXTRA_CLANG_ARGS="-I../../njs/src -I../../njs/build" \ make wasm-wasi-component if: steps.metadata.outputs.module == 'wasm-wasi-component' From 78145ff9662afa82013db3693d8cd73e8949bcb8 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 25 Sep 2026 21:32:49 +0200 Subject: [PATCH 246/391] ci(distros): run the standalone test binaries on the rolling legs #476 made the rolling legs build build/tests and the two standalone binaries, but nothing runs them, so musl still had compile coverage only. unit_port_recv_test moves control messages over socketpair()s with forked children, so it executes the NXT_CMSG_NXTHDR() loop #475 changed against musl's macro at runtime; nothing had ever done that. Run unit_close_test and unit_port_recv_test right after "make tests" in both fedora-rawhide and alpine-edge, on both compilers, with the same invocation as build-test's c-tests: no sudo, no capabilities. build/tests stays unrun there for the reason #476 gave -- its clone and credential cases need capabilities, and nxt_tests.c has no per-test filter. alpine-no-linux-headers is untouched; it tests the memfd probe and nothing else. Both binaries were run under the default docker seccomp profile in alpine:edge (gcc and clang) and fedora:rawhide (gcc) with the exact package sets and configure lines of these jobs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-distros.yml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 5d29c0b88..8775c5e9b 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -58,11 +58,20 @@ jobs: # --tests has always been configured here, but only "all" was built, # so a toolchain that breaks the tests binary alone went unnoticed - # (musl's CMSG_NXTHDR under clang, #475). Built, not run: the clone - # and credential cases need capabilities this container may not have. + # (musl's CMSG_NXTHDR under clang, #475). build/tests itself is built, + # not run: the clone and credential cases need capabilities this + # container may not have, and nxt_tests.c has no per-test filter. - name: make tests run: make -j 4 tests + # The two standalone binaries need no privileges and run here as they do + # in build-test's c-tests. See the same steps in alpine-edge for why. + - name: Run libunit close-provenance test + run: ./build/unit_close_test + + - name: Run libunit port_recv fd-ownership test + run: ./build/unit_port_recv_test + - name: configure unit-php run: ./configure php @@ -185,11 +194,22 @@ jobs: # --tests has always been configured here, but only "all" was built, # so a toolchain that breaks the tests binary alone went unnoticed - # (musl's CMSG_NXTHDR under clang, #475). Built, not run: the clone - # and credential cases need capabilities this container may not have. + # (musl's CMSG_NXTHDR under clang, #475). build/tests itself is built, + # not run: the clone and credential cases need capabilities this + # container may not have, and nxt_tests.c has no per-test filter. - name: make tests run: make -j 4 tests + # The two standalone binaries need no privileges, so they run here as + # they do in build-test's c-tests. unit_port_recv_test moves control + # messages over socketpair()s, so it executes the NXT_CMSG_NXTHDR() loop + # #475 changed against musl's macro; nothing else runs that on musl. + - name: Run libunit close-provenance test + run: ./build/unit_close_test + + - name: Run libunit port_recv fd-ownership test + run: ./build/unit_port_recv_test + - name: configure unit-php run: | ln -s /usr/lib/libphp83.so /usr/lib/libphp.so From 0736a5f7386cc987d0ada62553e8dabd6b7cd215 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 16:02:07 +0000 Subject: [PATCH 247/391] fix(h1): hold the listener configuration until the TLS connection is freed ASan found a heap-use-after-free in nxt_openssl_conn_io_shutdown() on an HTTP/1 keep-alive TLS connection that was closed after a reconfiguration. The TLS connection reads tls->conf->no_wait_shutdown during its shutdown. That nxt_tls_conf_t lives in the memory pool of the router configuration, and only a request held a reference to that configuration; the connection itself held none. The listener stays across most reconfigurations, so nxt_router_listen_socket_update() replaced its joint and released the old one at once. When the last request on the connection had already released its own reference, the old configuration was destroyed while the idle connection still pointed at its nxt_tls_conf_t. The next close of that connection, by the client or by idle_timeout, ran the TLS shutdown on freed memory. A request that is not keep-alive has the same order inside nxt_h1p_request_close(): the joint is released first, then nxt_h1p_shutdown() runs the TLS shutdown. The handshake has the same gap. A connection that has started TLS but has not made a request holds nothing, and the servername and session ticket callbacks read conf->bundle_hash, conf->bundle and conf->tickets from the same pool while the handshake waits for the client. The connection now takes a reference to its joint in nxt_http_conn_test(), before the TLS connection is created, and a cleanup on c->mem_pool releases it. That cleanup runs in nxt_conn_free(), after the TLS shutdown and after the handshake, so every read of tls->conf sees a live configuration. This is the same approach HTTP/2 uses. Copying no_wait_shutdown into the per-connection TLS state would have fixed only the shutdown read, not the handshake callbacks. A reference keeps an old configuration alive while a TLS connection lives, at most idle_timeout for an idle one. A deleted listener already waits for every accepted connection before its joint goes, so this changes nothing there; only a listener that stays keeps its old configuration a little longer. The reference is released before nxt_router_listen_event_release() checks engine->joints, so a worker engine still exits when its last connection is freed. Test test_reconfigure_tls_keepalive_close[client|idle_timeout]: a keep-alive TLS request, a reconfiguration that keeps the listener, then the client closes the connection, or idle_timeout does. Before the fix ASan reported a heap-use-after-free in nxt_openssl.c, freed by nxt_router_conf_release(), and the router was restarted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- src/nxt_h1proto.c | 36 +++++++++++++++++++++++++++++ test/test_reconfigure_tls.py | 44 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 748474694..f9a0150b8 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -21,6 +21,8 @@ #if (NXT_TLS) static ssize_t nxt_http_idle_io_read_handler(nxt_task_t *task, nxt_conn_t *c); static void nxt_http_conn_test(nxt_task_t *task, void *obj, void *data); +static void nxt_http_conn_tls_conf_release(nxt_task_t *task, void *obj, + void *data); #endif static ssize_t nxt_h1p_idle_io_read_handler(nxt_task_t *task, nxt_conn_t *c); static void nxt_h1p_conn_proto_init(nxt_task_t *task, void *obj, void *data); @@ -379,9 +381,43 @@ nxt_http_conn_test(nxt_task_t *task, void *obj, void *data) tls = joint->socket_conf->tls; + /* + * The connection holds the listener configuration until it is freed. + * The TLS connection reads its nxt_tls_conf_t, which lives in the memory + * pool of the router configuration, in the handshake callbacks and in + * the TLS shutdown. Only a request references the configuration, so + * without this reference a reconfiguration would destroy it under a + * connection in the handshake, and under a keep-alive connection that + * is closed after its last request has released its own reference. + * The cleanup runs in nxt_conn_free(), after the TLS shutdown. + */ + if (nxt_slow_path(nxt_mp_cleanup(c->mem_pool, + nxt_http_conn_tls_conf_release, + &engine->task, joint, NULL) + != NXT_OK)) + { + nxt_h1p_closing(task, c); + return; + } + + joint->count++; + tls->conn_init(task, tls, c); } + +static void +nxt_http_conn_tls_conf_release(nxt_task_t *task, void *obj, void *data) +{ + nxt_socket_conf_joint_t *joint; + + joint = obj; + + nxt_debug(task, "http conn tls conf release"); + + nxt_router_conf_release(task, joint); +} + #endif diff --git a/test/test_reconfigure_tls.py b/test/test_reconfigure_tls.py index 4f7d344ae..7530a169f 100644 --- a/test/test_reconfigure_tls.py +++ b/test/test_reconfigure_tls.py @@ -120,3 +120,47 @@ def test_reconfigure_tls_3(): clear_conf() assert client.get(sock=ssl_sock)['status'] == 408, 'request timeout' + + +@pytest.mark.parametrize('close', ['client', 'idle_timeout']) +def test_reconfigure_tls_keepalive_close(close): + # A keep-alive TLS connection that is closed after a reconfiguration. + # + # The listener stays, so the reconfiguration replaces its joint at once + # and releases the old one. Only the request held the old router + # configuration, and that reference went with the request. The idle + # connection's TLS state still points at the listener's nxt_tls_conf_t + # in that configuration, and the TLS shutdown at close read it: a + # heap-use-after-free under ASan, which the fixture's log check reports. + if close == 'idle_timeout': + assert 'success' in client.conf( + {'http': {'idle_timeout': 1}}, 'settings' + ) + + (resp, sock) = client.get_ssl( + headers={'Host': 'localhost', 'Connection': 'keep-alive'}, + start=True, + read_timeout=1, + ) + + assert resp['status'] == 200, 'keep-alive request' + + assert 'success' in client.conf({'return': 201}, 'routes/0/action') + + if close == 'client': + sock.close() + + else: + # The router closes the idle connection itself. + sock.settimeout(5) + + try: + assert sock.recv(1) == b'', 'closed by the router' + except OSError: + pass + + # Let the router run the TLS shutdown of the closed connection; it must + # still be alive and serve the new configuration. + time.sleep(0.5) + + assert client.get_ssl()['status'] == 201, 'router alive' From 585ba7cb35e91ad62d399b7ad78be0b1d5f08cc2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 16:11:50 +0000 Subject: [PATCH 248/391] test(tls): read to EOF when the router closes an idle connection The router writes a 408 response before it closes an idle connection (nxt_h1p_idle_response()), so the first recv() can return data, not EOF, and the idle_timeout case failed now and then with 'closed by the router'. Read until EOF instead. It still catches the TLS shutdown use-after-free under ASan on master. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- test/test_reconfigure_tls.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_reconfigure_tls.py b/test/test_reconfigure_tls.py index 7530a169f..2d655c8f2 100644 --- a/test/test_reconfigure_tls.py +++ b/test/test_reconfigure_tls.py @@ -151,11 +151,13 @@ def test_reconfigure_tls_keepalive_close(close): sock.close() else: - # The router closes the idle connection itself. + # The router closes the idle connection itself. It may write a + # 408 response first (nxt_h1p_idle_response()), so read up to EOF. sock.settimeout(5) try: - assert sock.recv(1) == b'', 'closed by the router' + while sock.recv(4096) != b'': + pass except OSError: pass From f9a9045e758584565c9f47a76aaeb0adf3763351 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 16:16:32 +0000 Subject: [PATCH 249/391] test(tls): fail when the router never closes the idle connection socket.timeout is an OSError, so a connection that the idle timeout never closed was accepted as closed, and the test could pass without the TLS shutdown path it is meant to cover. Fail on TimeoutError; keep other OSErrors (an SSL error or a reset) as a close. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- test/test_reconfigure_tls.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_reconfigure_tls.py b/test/test_reconfigure_tls.py index 2d655c8f2..291000ca0 100644 --- a/test/test_reconfigure_tls.py +++ b/test/test_reconfigure_tls.py @@ -158,7 +158,10 @@ def test_reconfigure_tls_keepalive_close(close): try: while sock.recv(4096) != b'': pass + except TimeoutError: + pytest.fail('the router did not close the idle connection') except OSError: + # An SSL error or a reset: the router closed it. pass # Let the router run the TLS shutdown of the closed connection; it must From 5bfe284b8abd36e65b744bf8eca6cfaf40d6fbbb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 16:03:29 +0000 Subject: [PATCH 250/391] fix(static): close the file before the request pool can go nxt_http_static_buf_completion() on its "clean" path (the request has failed, or the file read failed) first freed the buffers and released the request pool reference that each of them holds, and only then closed the file, cleared r->out and freed the file buffer. When those buffers hold the last references, because the request has already closed, the pool is destroyed inside that loop. Its cleanup, nxt_http_static_buf_cleanup(), finds r->out still set and closes the file itself. The code after the loop then closes the same descriptor again, which fails with EBADF, or closes a descriptor that another connection got in the meantime, and it writes r->out in the freed request. The file is now closed and r->out cleared before the buffers are released. Over HTTP/1 this is defensive. Every HTTP/1 error path (a write error, a peer close, send_timeout, a failed file read) goes through the request's discard: it drains c->write, so the static buffers complete before the last buffer's completion closes the request and releases the request's own pool reference. A test with 60 aborted transfers of a 32 MB file (client RST with a small receive buffer, half-close, and send_timeout) under ASan logged no alert and no report. HTTP/2 reaches the path: a response header is submitted and the static body handler is queued, then a connection error in the same read fails the request, and the request closes before its body buffers complete. The code is shared, so the order is fixed here for both. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- src/nxt_http_static.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 506200237..6954ad834 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -1861,6 +1861,18 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) clean: + /* + * The file goes first: the buffers below may hold the last references + * to the request pool, and when the pool is destroyed r is gone and + * nxt_http_static_buf_cleanup() would close the file a second time. + */ + if (fb != NULL) { + nxt_file_close(task, fb->file); + r->out = NULL; + + nxt_http_static_buf_free(fb); + } + do { next = b->next; @@ -1869,13 +1881,6 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) b = next; } while (b != NULL); - - if (fb != NULL) { - nxt_file_close(task, fb->file); - r->out = NULL; - - nxt_http_static_buf_free(fb); - } } From 96da3b67126cc1c3c3ed11e0f7ac73c275d155db Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 12:58:47 +0000 Subject: [PATCH 251/391] fix(proxy): send one Content-Length for a chunked request body A request body without a length (HTTP/1 chunked, or HTTP/2 with no content-length) is read whole, and nxt_http_request_chunked_transform() adds a Content-Length field to the request fields. The upstream request header copies those fields, and nxt_h1p_peer_header_send() then added a Content-Length of its own for r->chunked. The upstream got two Content-Length fields, and a Unit upstream answers 400 to that. Add the computed field only when the transform has not set r->content_length. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- src/nxt_h1proto.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index f9a0150b8..66e82d75f 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -2529,9 +2529,14 @@ nxt_h1p_peer_header_send(nxt_task_t *task, nxt_http_peer_t *peer) + sizeof("Connection: close\r\n") + sizeof("\r\n"); - /* Emit Content-Length after chunked_transform; NULL body → value 0. */ + /* + * Emit Content-Length after chunked_transform; NULL body → value 0. + * The transform adds a Content-Length field (r->content_length) that + * goes out with the other fields; a second one would make the + * upstream answer 400. + */ content_length = -1; - if (r->chunked) { + if (r->chunked && r->content_length == NULL) { if (r->body == NULL) { content_length = 0; } else { From c046d9534cfe51e9dffd39d2a53accef9992c323 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 13:19:37 +0000 Subject: [PATCH 252/391] test(proxy): check that a chunked request goes upstream with one Content-Length The upstream in this test counts the Content-Length fields it gets and returns the count. Without the previous commit it gets 2. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- test/test_proxy_chunked.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/test_proxy_chunked.py b/test/test_proxy_chunked.py index 0f349ddb8..75e1df2d1 100644 --- a/test/test_proxy_chunked.py +++ b/test/test_proxy_chunked.py @@ -648,3 +648,32 @@ def worker(body_bytes): finally: proc.terminate() proc.wait() + + +def _handler_counts_content_length(data): + headers = data.split('\r\n\r\n', 1)[0].split('\r\n')[1:] + count = sum(1 for h in headers if h.lower().startswith('content-length:')) + body = str(count) + return ( + "HTTP/1.1 200 OK\r\nConnection: close\r\n" + f"Content-Length: {len(body)}\r\n\r\n{body}" + ) + + +def test_proxy_chunked_request_one_content_length(): + """A chunked request body goes upstream with exactly one Content-Length.""" + port = _get_free_port() + _run_proxy_server(_handler_counts_content_length, port) + _configure_proxy(port, 'counting backend') + + resp = client.post( + headers={ + 'Host': 'localhost', + 'Transfer-Encoding': 'chunked', + 'Connection': 'close', + }, + body=_chunked_encode(b'0123456789').decode('latin-1'), + ) + + assert resp['status'] == 200, 'status' + assert resp['body'] == '1', 'one Content-Length upstream' From 2d1fea2ef69a2138ee56fefdf9a73525a82e7cf7 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 21 Sep 2026 09:19:59 +0200 Subject: [PATCH 253/391] java: send a complete WebSocket text message as one frame The blocking text path encoded every message through one fixed 8 KiB buffer and emitted a frame per buffer fill, so a 64 KiB message left as eight frames and a 16 MiB one as about two thousand, each carrying its own frame header and costing its own message to the router. "max_frame_size" has no say in this: the key is only an inbound limit in the router (src/nxt_h1proto_websocket.c:360-368) and is never sent to an application. Every other language module sends a complete message as a single frame. A message that does not fit in encoderBuffer is now encoded into a buffer sized from an exact pre-scan of its UTF-8 length, up to MAX_SEND_BUFFER_SIZE: 16 MiB, overridable through the nginx.unit.websocket.MAX_SEND_BUFFER_SIZE system property, the same idiom as the existing DEFAULT_BUFFER_SIZE. That buffer is a local, so a session that once sent a large message does not go on holding a large buffer, and the 8 KiB encoderBuffer field still serves every message below it with no allocation at all. The send loop is unchanged, so a message above the cap fragments exactly as it did. The cap comparison is inclusive, and that is load-bearing. Autobahn case 9.1.6 sends 16777216 bytes, which is the default cap to the byte; an exclusive test would re-fragment the largest case into 2048 frames and fail with the same fin mismatch this commit exists to remove. The buffer is on the heap rather than direct. The reasoning is code reading, not a benchmark, and no GC or throughput numbers were taken: - encoderBuffer is itself ByteBuffer.allocate(), so a heap buffer keeps a large message on the JNI path that every text frame already takes, nxt_java_Request_sendWsFrameArr (src/java/nxt_jni_Request.c:742). The direct-buffer path at :715 is not equivalent code; among other things it has no nxt_unit_response_send() fallback. That fallback is unreachable today, because nxt_java_service sends the response before entering WebSocket mode (src/nxt_java.c:530-536), but there is no reason to make large messages the only text sends that take the other branch. - Utf8Encoder takes encodeHasArray() only when both buffers have an accessible array. A direct output buffer would push array-backed input, WsWriter's among it, onto the per-character encodeNotHasArray() loop for precisely the largest messages. - A direct allocation is off-heap, bounded by -XX:MaxDirectMemorySize and reclaimed only through a Cleaner, which is a poor fit for a buffer that lives for the duration of one call. The longer GetPrimitiveArrayCritical region is not a regression: one 16 MiB send holds the region across one copy into shared memory and two port messages, where 2048 sends held 2048 regions, each with its own copy and its own port message. An unpaired surrogate still throws IllegalArgumentException, and under the cap it now throws before anything is on the wire instead of after N frames with fin=0 have already left. The pre-scan charges such a character three bytes, which is arbitrary and never observable: Utf8Encoder reports every unpaired shape as a coder error, either malformedForLength(1) inside the buffer or UNDERFLOW at its very end, which CharsetEncoder.encode(in, out, true) converts to malformed input under the default REPORT action. encoder.reset() is still not required here: encode(in, out, true) is permitted in ST_END and nothing on the blocking path calls flush(). WsWriter is affected through the shared method rather than untouched: its 8192-character chunks of multi-byte text used to leave as two or three frames and now leave as one. Each chunk is still a frame of its own, so the streaming meaning is unchanged. WsOutputStream is binary, never reaches the encoder, and is genuinely untouched. The async path is left alone because it writes nothing in this fork - WsRemoteEndpointImplServer.doWrite() is an empty stub - which is a separate defect. Co-Authored-By: Claude Opus 5 --- src/java/nginx/unit/websocket/Constants.java | 9 +++ .../websocket/WsRemoteEndpointImplBase.java | 80 ++++++++++++++++++- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/src/java/nginx/unit/websocket/Constants.java b/src/java/nginx/unit/websocket/Constants.java index 38b22fe08..bef5bb399 100644 --- a/src/java/nginx/unit/websocket/Constants.java +++ b/src/java/nginx/unit/websocket/Constants.java @@ -45,6 +45,15 @@ public class Constants { "nginx.unit.websocket.DEFAULT_BUFFER_SIZE", 8 * 1024) .intValue(); + /* + * The largest text message that is encoded into a buffer of its own, and + * therefore sent as a single frame. A larger message is encoded through + * DEFAULT_BUFFER_SIZE and fragments. + */ + static final int MAX_SEND_BUFFER_SIZE = Integer.getInteger( + "nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 * 1024 * 1024) + .intValue(); + // Client connection /** * Property name to set to configure the value that is passed to diff --git a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java index d451db7d2..d335f9bf9 100644 --- a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java +++ b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java @@ -240,18 +240,90 @@ public Writer getSendWriter() { } + private static final long OVER_CAP = -1; + + /* + * The exact UTF-8 encoded length of part, without consuming it, or + * OVER_CAP once the length passes cap. The scan stops at the cap, so an + * oversized message does not pay for a full pass. + * + * The widths below are Utf8Encoder's own: one byte up to 0x7F, two up to + * 0x7FF, four for a high surrogate followed by a low one, three for the + * rest. An unpaired surrogate lands in the last branch, and the width + * charged for it there is arbitrary, because such a message never reaches + * the wire: Utf8Encoder returns malformedForLength(1) for an unpaired + * surrogate inside the buffer, and UNDERFLOW for one at its very end, + * which CharsetEncoder.encode(in, out, true) turns into malformed input + * under the default REPORT action. Either way the encode below reports an + * error and throws. + */ + private static long utf8Length(CharBuffer part, long cap) { + long len = 0; + int limit = part.limit(); + + for (int i = part.position(); i < limit; i++) { + char c = part.get(i); + + if (c < 0x80) { + len += 1; + } else if (c < 0x800) { + len += 2; + } else if (Character.isHighSurrogate(c) && i + 1 < limit + && Character.isLowSurrogate(part.get(i + 1))) + { + len += 4; + i++; + } else { + len += 3; + } + + /* + * Inclusive, and load-bearing: a message of exactly cap bytes + * still gets a buffer of its own. Autobahn case 9.1.6 sends + * 16777216 bytes, which is the default cap to the byte, so a + * ">=" here would fragment the largest case back into 2048 frames + * and fail with the very fin mismatch this method exists to fix. + */ + if (len > cap) { + return OVER_CAP; + } + } + + return len; + } + + void sendMessageBlock(CharBuffer part, boolean last) throws IOException { long timeoutExpiry = getTimeoutExpiry(); boolean isDone = false; + + /* + * Encode the whole message into one buffer where it fits under the + * cap, so that it leaves as a single frame. Above the cap, and for + * the messages that already fit, encoderBuffer is used as before; the + * loop is unchanged, so an oversized message still fragments exactly + * as it did. + * + * The buffer is on the heap and not direct, to keep the large message + * on the same JNI path as every message that fits in encoderBuffer, + * which is itself a heap buffer. + */ + ByteBuffer buffer = encoderBuffer; + long needed = utf8Length(part, Constants.MAX_SEND_BUFFER_SIZE); + + if (needed != OVER_CAP && needed > encoderBuffer.capacity()) { + buffer = ByteBuffer.allocate((int) needed); + } + while (!isDone) { - encoderBuffer.clear(); - CoderResult cr = encoder.encode(part, encoderBuffer, true); + buffer.clear(); + CoderResult cr = encoder.encode(part, buffer, true); if (cr.isError()) { throw new IllegalArgumentException(cr.toString()); } isDone = !cr.isOverflow(); - encoderBuffer.flip(); - sendMessageBlock(Constants.OPCODE_TEXT, encoderBuffer, last && isDone, timeoutExpiry); + buffer.flip(); + sendMessageBlock(Constants.OPCODE_TEXT, buffer, last && isDone, timeoutExpiry); } stateMachine.complete(last); } From 05820ad1311d09d51d58770ef336e233cfd14858 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 21 Sep 2026 09:20:12 +0200 Subject: [PATCH 254/391] tests: raise the java websockets mirror receive buffers The mirror app accepted an 8 MiB text message and, by omission, only an 8 KiB fragmented binary one, so two groups of Autobahn cases closed the connection with 1009 before the module under test was reached: - 9.1.6 sends 16 MiB of text. messageBufferText is a CharBuffer of maxTextMessageBufferSize characters (WsSession.java:882-884), 8388608 of them, and no MessageHandler.Partial is registered, so the message overflowed. - 9.4.1 to 9.4.9 send 4 MiB of binary in fragments. A fragmented binary message takes the accumulating branch and allocates binaryBuffer at maxBinaryMessageBufferSize (WsSession.java:985-993), which the app never set, leaving the 8 KiB default. An unfragmented binary message is handed over whole and never touches that buffer, which is why 9.2.1 to 9.2.6 passed at 16 MiB regardless. Both buffers are now 16 MiB, which is what the largest case needs. This is a fixture limit and not the module defect fixed in the previous commit, and neither change covers for the other. Measured separately: with the module fix alone, 9.1.6 and 9.4.x still close with 1009; with this fixture change alone, 9.1.1 still fails on frame opcode=0 fin=False len=8192. Co-Authored-By: Claude Opus 5 --- test/java/websockets_mirror/app.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/java/websockets_mirror/app.java b/test/java/websockets_mirror/app.java index ada60231b..deeb0c9c8 100644 --- a/test/java/websockets_mirror/app.java +++ b/test/java/websockets_mirror/app.java @@ -12,7 +12,8 @@ public class app { @OnOpen public void onOpen(Session session) { - session.setMaxTextMessageBufferSize(8388608); + session.setMaxTextMessageBufferSize(16 * 1024 * 1024); + session.setMaxBinaryMessageBufferSize(16 * 1024 * 1024); } @OnMessage From a0a0e61f63c3ea883dc5fab63d994383bec9410f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 21 Sep 2026 09:33:05 +0200 Subject: [PATCH 255/391] changelog: a complete java websocket text message is sent as one frame Co-Authored-By: Claude Opus 5 --- CHANGES | 13 +++++++++++++ docs/changes.xml | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/CHANGES b/CHANGES index ed2f860c2..00e3ff858 100644 --- a/CHANGES +++ b/CHANGES @@ -401,6 +401,19 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 key and the TrueAsync test suite are removed with it; a PHP application builds and behaves as before. + *) Bugfix: the java module split every outgoing WebSocket text message + larger than 8 KiB into 8 KiB frames, whatever "max_frame_size" was set + to. A 64 KiB message arrived as eight frames and a 16 MiB one as about + two thousand, each carrying its own frame header and costing its own + message to the router, while every other language module sends a + complete message as a single frame. A complete text message is now + encoded into a buffer of its own and sent as one frame; a message + above "nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 MiB by default, + still fragments. A Writer from getSendWriter() still sends each chunk + it flushes as a frame of its own, but that chunk is now one frame + instead of up to three. This does not make the module honour + "max_frame_size", which remains an inbound limit only. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index 9313b7489..028a05354 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -530,6 +530,22 @@ application builds and behaves as before. + + +the java module split every outgoing WebSocket text message larger than 8 KiB +into 8 KiB frames, whatever "max_frame_size" was set to. A 64 KiB message +arrived as eight frames and a 16 MiB one as about two thousand, each carrying +its own frame header and costing its own message to the router, while every +other language module sends a complete message as a single frame. A complete +text message is now encoded into a buffer of its own and sent as one frame; a +message above "nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 MiB by default, +still fragments. A Writer from getSendWriter() still sends each chunk it +flushes as a frame of its own, but that chunk is now one frame instead of up to +three. This does not make the module honour "max_frame_size", which remains an +inbound limit only. + + + From ff170d87efde840cead9736189f581b36882ea72 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 22:48:50 +0000 Subject: [PATCH 256/391] java: send through the asynchronous WebSocket remote getAsyncRemote() sent nothing. WsRemoteEndpointImplServer.doWrite() was an empty stub, so every async send -- sendText(), sendBinary() and sendObject(), with a SendHandler or through the Future they return -- wrote no frame, never called its handler, and left Future.get() blocked for good. getBasicRemote() sends through another path and was not affected. doWrite() now sends what the base class serialised through Request.sendWsFrame(): the direct header and payload, or the batching output buffer, where a frame may straddle calls and a small stream decoder picks it apart. It completes the handler on the sending thread with the outcome, a failure on error or once the session is closed, and a send that fails partway resets the decoder. A handler that sends from onResult(), as TextMessageSendHandler does for each 8 KiB of a text message, nests a completion inside a completion. Nested completions run inline up to 32 deep, so a handler can wait on a send it made itself (get(), flushBatch(), close() with batching on), and are queued beyond that and run by the outermost completion on the thread, so a 16 MiB message does not overflow the stack. The state is per thread: once a completion releases the message part, another thread may send on the endpoint while the first is still completing. Three defects behind the stub surfaced once sends went out: - startMessage() dereferenced a transformation this fork never installs, since Unit parses frames itself and creates no WsFrameServer; it now skips a null one. - flushBatch() handed sendWsFrame() a null payload and threw, so disallowing batching failed and batched messages were never sent although their futures reported success. The flush now goes through startMessage(), which hands the output buffer to doWrite(). - writeHeader() wrote payload.limit() as the frame length, but both write paths send position()..limit(); a buffer that does not start at 0 got a header that overstated its frame, and on the batching path every later frame was corrupt. It now writes remaining(). Async text still leaves in 8 KiB frames; the blocking path's single-frame change is #433. Request.sendWsFrame() ignoring arrayOffset() is #486. Tests: a websockets_async mirror app sending through the async remote, with cases for text, binary, 64 KiB and 16 MiB (--unsafe), the SendHandler and Future forms and a failed future, batching, batching from a buffer at a non-zero position, eight threads sending at once, and a handler that waits on its own send and flushes a batch. Closes #434. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 14 + docs/changes.xml | 16 ++ .../websocket/WsRemoteEndpointImplBase.java | 74 ++++- .../websocket/server/LocalStrings.properties | 2 + .../server/WsRemoteEndpointImplServer.java | 242 ++++++++++++++++ test/java/websockets_async/app.java | 265 ++++++++++++++++++ test/test_java_websockets.py | 214 ++++++++++++++ 7 files changed, 816 insertions(+), 11 deletions(-) create mode 100644 test/java/websockets_async/app.java diff --git a/CHANGES b/CHANGES index 00e3ff858..106de7df4 100644 --- a/CHANGES +++ b/CHANGES @@ -414,6 +414,20 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 instead of up to three. This does not make the module honour "max_frame_size", which remains an inbound limit only. + *) Bugfix: the java module sent nothing through the asynchronous + WebSocket remote. Every send from getAsyncRemote() -- sendText(), + sendBinary() and sendObject(), with a SendHandler or through the + Future they return -- ended in a doWrite() that had no body, so no + frame left, the handler was never called and Future.get() blocked + for good; getBasicRemote() sends through another path and was not + affected. Such a send now goes out as one from the blocking remote + does, and its handler or future completes on the sending thread with + the outcome, which is a failure once the session is closed (#434). + Batching works as well: flushBatch() and setBatchingAllowed(false) + threw instead of sending what was batched, and a batched frame from a + buffer whose position was not 0 had a header that overstated its + length and corrupted the frames after it. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index 028a05354..1fcb94494 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -546,6 +546,22 @@ inbound limit only. + + +the java module sent nothing through the asynchronous WebSocket remote. Every +send from getAsyncRemote() -- sendText(), sendBinary() and sendObject(), with a +SendHandler or through the Future they return -- ended in a doWrite() that had +no body, so no frame left, the handler was never called and Future.get() +blocked for good; getBasicRemote() sends through another path and was not +affected. Such a send now goes out as one from the blocking remote does, and +its handler or future completes on the sending thread with the outcome, which +is a failure once the session is closed (#434). Batching works as well: +flushBatch() and setBatchingAllowed(false) threw instead of sending what was +batched, and a batched frame from a buffer whose position was not 0 had a +header that overstated its length and corrupted the frames after it. + + + diff --git a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java index d335f9bf9..6d4ad509a 100644 --- a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java +++ b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java @@ -28,9 +28,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Queue; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import javax.websocket.CloseReason; @@ -120,9 +122,27 @@ public boolean getBatchingAllowed() { } + /* + * The batched frames sit in outputBuffer, which only writeMessagePart() + * knows how to hand to doWrite(), so the flush goes through startMessage() + * like any other message part. sendMessageBlock() writes straight to the + * request instead, and with no payload it has nothing to write. + */ @Override public void flushBatch() throws IOException { - sendMessageBlock(Constants.INTERNAL_OPCODE_FLUSH, null, true); + FutureToSendHandler f2sh = new FutureToSendHandler(wsSession); + startMessage(Constants.INTERNAL_OPCODE_FLUSH, null, true, f2sh); + + try { + long timeout = getBlockingSendTimeout(); + if (timeout < 0) { + f2sh.get(); + } else { + f2sh.get(timeout, TimeUnit.MILLISECONDS); + } + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new IOException(e); + } } @@ -379,7 +399,16 @@ void startMessage(byte opCode, ByteBuffer payload, boolean last, intermediateMessageHandler, new EndMessageHandler(this, handler), -1)); - messageParts = transformation.sendMessagePart(messageParts); + /* + * On the server side there is no transformation: Tomcat's + * WsHttpUpgradeHandler.init() takes it from the WsFrameServer it + * creates, and Unit parses the frames itself, so no WsFrameServer is + * ever created and no transformation is installed. The message + * parts then go out as they are. + */ + if (transformation != null) { + messageParts = transformation.sendMessagePart(messageParts); + } // Some extensions/transformations may buffer messages so it is possible // that no message parts will be returned. If this is the case the @@ -714,6 +743,22 @@ protected void setRequest(Request request) { this.request = request; } + + /* + * The transport doWrite() sends through. Unit writes the frame header + * itself, so a frame is handed over as its payload, opcode and fin bit + * rather than as the serialised header writeHeader() built. + */ + protected final void sendWsFrame(ByteBuffer payload, byte opCode, + boolean fin, long timeoutExpiry) throws IOException { + request.sendWsFrame(payload, opCode, fin, timeoutExpiry); + } + + + protected final boolean isSessionOpen() { + return wsSession.isOpen(); + } + protected void setEncoders(EndpointConfig endpointConfig) throws DeploymentException { encoderEntries.clear(); @@ -786,13 +831,20 @@ private static void writeHeader(ByteBuffer headerBuffer, boolean fin, b = 0; } + /* + * The frame carries position()..limit(), which is what both write + * paths send; limit() alone overstates it for a buffer that does not + * start at 0, such as a slice or a partly read one. + */ + int len = payload.remaining(); + // Next write the mask && length length - if (payload.limit() < 126) { - headerBuffer.put((byte) (payload.limit() | b)); - } else if (payload.limit() < 65536) { + if (len < 126) { + headerBuffer.put((byte) (len | b)); + } else if (len < 65536) { headerBuffer.put((byte) (126 | b)); - headerBuffer.put((byte) (payload.limit() >>> 8)); - headerBuffer.put((byte) (payload.limit() & 0xFF)); + headerBuffer.put((byte) (len >>> 8)); + headerBuffer.put((byte) (len & 0xFF)); } else { // Will never be more than 2^31-1 headerBuffer.put((byte) (127 | b)); @@ -800,10 +852,10 @@ private static void writeHeader(ByteBuffer headerBuffer, boolean fin, headerBuffer.put((byte) 0); headerBuffer.put((byte) 0); headerBuffer.put((byte) 0); - headerBuffer.put((byte) (payload.limit() >>> 24)); - headerBuffer.put((byte) (payload.limit() >>> 16)); - headerBuffer.put((byte) (payload.limit() >>> 8)); - headerBuffer.put((byte) (payload.limit() & 0xFF)); + headerBuffer.put((byte) (len >>> 24)); + headerBuffer.put((byte) (len >>> 16)); + headerBuffer.put((byte) (len >>> 8)); + headerBuffer.put((byte) (len & 0xFF)); } if (masked) { headerBuffer.put(mask[0]); diff --git a/src/java/nginx/unit/websocket/server/LocalStrings.properties b/src/java/nginx/unit/websocket/server/LocalStrings.properties index 5bc125017..4fd7b2810 100644 --- a/src/java/nginx/unit/websocket/server/LocalStrings.properties +++ b/src/java/nginx/unit/websocket/server/LocalStrings.properties @@ -41,3 +41,5 @@ wsHttpUpgradeHandler.noPreInit=The preInit() method must be called to configure wsHttpUpgradeHandler.serverStop=The server is stopping wsRemoteEndpointServer.closeFailed=Failed to close the ServletOutputStream connection cleanly +wsRemoteEndpointServer.closed=Message will not be sent because the WebSocket session has been closed +wsRemoteEndpointServer.badFrame=The output buffer holds a frame of [{0}] bytes, more than a message may carry diff --git a/src/java/nginx/unit/websocket/server/WsRemoteEndpointImplServer.java b/src/java/nginx/unit/websocket/server/WsRemoteEndpointImplServer.java index 6d10a3bee..c4f1f4428 100644 --- a/src/java/nginx/unit/websocket/server/WsRemoteEndpointImplServer.java +++ b/src/java/nginx/unit/websocket/server/WsRemoteEndpointImplServer.java @@ -22,6 +22,8 @@ import java.nio.ByteBuffer; import java.nio.channels.CompletionHandler; import java.nio.channels.InterruptedByTimeoutException; +import java.util.ArrayDeque; +import java.util.Queue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; @@ -50,6 +52,38 @@ public class WsRemoteEndpointImplServer extends WsRemoteEndpointImplBase { private volatile long timeoutExpiry = -1; private volatile boolean close; + /* + * A frame that the batching path split across two doWrite() calls: the + * base class fills its output buffer and flushes it wherever it happens + * to be full, so a frame's header, or its payload, may arrive in parts. + * A server frame is not masked, so its header is at most 10 bytes. + */ + private final byte[] header = new byte[10]; + private int headerLength = 0; + private byte frameOpCode; + private boolean frameFin; + private long frameLeft = 0; + private ByteBuffer framePayload = null; + + /* + * A completion runs on the thread that sent. A handler that sends from + * onResult(), as TextMessageSendHandler does for each 8 KiB of a text + * message, nests a doWrite() inside the completion, so a large message + * would nest thousands deep. Completions nest inline up to MAX_DEPTH and + * are queued beyond it, to be run by the outermost completion on this + * thread. Inline, not always queued: a handler that waits on a send it + * made itself (Future.get(), flushBatch(), close() with batching on) needs + * that send's completion to run before it returns. + * + * The state is per thread: once a completion releases the message part, + * another thread may send on this endpoint while the first is still + * inside its completion. + */ + private static final int MAX_DEPTH = 32; + + private static final ThreadLocal completions = + new ThreadLocal<>(); + public WsRemoteEndpointImplServer( WsServerContainer serverContainer) { } @@ -60,9 +94,217 @@ protected final boolean isMasked() { return false; } + /* + * Sends the frames the base class serialised into buffers. Unit writes + * the frame header itself, so each frame is decoded back into its opcode, + * fin bit and payload and handed to sendWsFrame(). + * + * writeMessagePart() sends a frame directly as its header and its payload, + * in that order. With batching allowed it copies frames into its output + * buffer instead and passes that buffer whenever it fills or is flushed, + * so that path is decoded as a byte stream that may split a frame across + * calls. + * + * The write is synchronous: sendWsFrame() has copied the payload to the + * router by the time it returns, so the handler is completed here, on the + * calling thread, with the outcome. + */ @Override protected void doWrite(SendHandler handler, long blockingWriteTimeoutExpiry, ByteBuffer... buffers) { + SendResult result = SENDRESULT_OK; + + try { + if (!isSessionOpen()) { + throw new IOException(sm.getString("wsRemoteEndpointServer.closed")); + } + + if (buffers.length == 2 && headerLength == 0 && frameLeft == 0) { + // Direct path: the frame's header, then its payload. + ByteBuffer hdr = buffers[0]; + ByteBuffer payload = buffers[1]; + byte b = hdr.get(hdr.position()); + + sendWsFrame(payload, (byte) (b & 0x0F), (b & 0x80) != 0, + blockingWriteTimeoutExpiry); + + hdr.position(hdr.limit()); + payload.position(payload.limit()); + } else { + for (ByteBuffer buffer : buffers) { + sendFrames(buffer, blockingWriteTimeoutExpiry); + } + } + } catch (IOException | RuntimeException e) { + /* + * A frame the stream decoder had started is lost with the send; + * start the next doWrite() at a frame boundary. + */ + headerLength = 0; + frameLeft = 0; + framePayload = null; + + result = new SendResult(e); + } + + complete(handler, result); + } + + + private void sendFrames(ByteBuffer buffer, long timeoutExpiry) + throws IOException { + while (buffer.hasRemaining()) { + if (frameLeft == 0 && framePayload == null) { + if (!readHeader(buffer)) { + // The rest of the header is in a later buffer + return; + } + if (frameLeft == 0) { + sendWsFrame(ByteBuffer.allocate(0), frameOpCode, frameFin, + timeoutExpiry); + continue; + } + } + + int limit = buffer.limit(); + + if (framePayload == null && buffer.remaining() >= frameLeft) { + // The whole payload is here: send it in place + buffer.limit(buffer.position() + (int) frameLeft); + sendWsFrame(buffer, frameOpCode, frameFin, timeoutExpiry); + buffer.position(buffer.limit()); + buffer.limit(limit); + frameLeft = 0; + continue; + } + + // The payload is split across buffers: gather it + if (framePayload == null) { + framePayload = ByteBuffer.allocate((int) frameLeft); + } + + int n = Math.min(buffer.remaining(), framePayload.remaining()); + buffer.limit(buffer.position() + n); + framePayload.put(buffer); + buffer.limit(limit); + frameLeft -= n; + + if (frameLeft == 0) { + framePayload.flip(); + ByteBuffer payload = framePayload; + framePayload = null; + sendWsFrame(payload, frameOpCode, frameFin, timeoutExpiry); + } + } + } + + + /* + * Reads header bytes from buffer until the header is complete. Returns + * false when buffer ran out first; the bytes read so far are kept for the + * next call. + */ + private boolean readHeader(ByteBuffer buffer) throws IOException { + while (buffer.hasRemaining()) { + header[headerLength++] = buffer.get(); + + if (headerLength < 2) { + continue; + } + + int len = header[1] & 0x7F; + int need = len == 126 ? 4 : len == 127 ? 10 : 2; + + if (headerLength < need) { + continue; + } + + frameFin = (header[0] & 0x80) != 0; + frameOpCode = (byte) (header[0] & 0x0F); + + if (len == 126) { + frameLeft = ((header[2] & 0xFF) << 8) | (header[3] & 0xFF); + } else if (len == 127) { + frameLeft = 0; + for (int i = 2; i < 10; i++) { + frameLeft = (frameLeft << 8) | (header[i] & 0xFF); + } + if (frameLeft < 0 || frameLeft > Integer.MAX_VALUE) { + headerLength = 0; + throw new IOException(sm.getString( + "wsRemoteEndpointServer.badFrame", + Long.valueOf(frameLeft))); + } + } else { + frameLeft = len; + } + + headerLength = 0; + return true; + } + + return false; + } + + + private void complete(SendHandler handler, SendResult result) { + Completions c = completions.get(); + + if (c == null) { + c = new Completions(); + completions.set(c); + + } else if (c.depth >= MAX_DEPTH) { + c.queue.add(new Completion(handler, result)); + return; + } + + boolean outermost = c.depth == 0; + + c.depth++; + try { + handler.onResult(result); + } finally { + c.depth--; + + if (outermost) { + drain(c); + } + } + } + + + private static void drain(Completions c) { + try { + Completion q; + while ((q = c.queue.poll()) != null) { + c.depth++; + try { + q.handler.onResult(q.result); + } finally { + c.depth--; + } + } + } finally { + completions.remove(); + } + } + + + private static class Completions { + private int depth = 0; + private final Queue queue = new ArrayDeque<>(); + } + + + private static class Completion { + private final SendHandler handler; + private final SendResult result; + + private Completion(SendHandler handler, SendResult result) { + this.handler = handler; + this.result = result; + } } @Override diff --git a/test/java/websockets_async/app.java b/test/java/websockets_async/app.java new file mode 100644 index 000000000..a3267ed9c --- /dev/null +++ b/test/java/websockets_async/app.java @@ -0,0 +1,265 @@ +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import javax.websocket.OnMessage; +import javax.websocket.OnOpen; +import javax.websocket.RemoteEndpoint; +import javax.websocket.SendHandler; +import javax.websocket.SendResult; +import javax.websocket.Session; +import javax.websocket.server.ServerEndpoint; + +/* + * A mirror app that echoes through the asynchronous remote (issue #434). + * + * A text message that starts with "handler:" is echoed with sendText(String, + * SendHandler); one that starts with "object:" with sendObject(Object), as an + * Integer when the rest of it is one and as a String otherwise; any other + * text message with the Future from sendText(String). A binary message + * whose first byte is 'h' is echoed with sendBinary(ByteBuffer, SendHandler), + * any other with the Future from sendBinary(ByteBuffer). + * + * After the echo the app reports what the asynchronous API told it, as a text + * frame sent through the blocking remote: "handler-ok" or "handler-fail: ..." + * once the SendHandler has run, "future-done" once Future.get() has returned, + * "future-timeout" when it has not returned within five seconds, or + * "future-fail: ..." when it threw. So a client sees exactly one of the two + * defects of #434 when the echo is missing, and the other when the report is. + * + * "batch:N" allows batching, sends N messages "b0".."b" through the + * asynchronous remote, then disallows batching again, which flushes them, and + * reports "batch-flushed" or "batch-fail: ...". "threads:N" sends "t0".."t" + * from N threads at once and reports "threads-done" once every future has + * completed, or the first failure or timeout. A binary message whose first + * byte is 'o' is sent batched from its fourth byte on; see offset(). + * "nested:..." waits on sends from inside a SendHandler; see nested(). + */ +@ServerEndpoint("/") +public class app { + + @OnOpen + public void onOpen(Session session) { + session.setMaxTextMessageBufferSize(16 * 1024 * 1024); + session.setMaxBinaryMessageBufferSize(16 * 1024 * 1024); + } + + @OnMessage + public void echoTextMessage(Session session, String msg) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + if (msg.startsWith("nested:")) { + nested(session, msg); + } else if (msg.startsWith("batch:")) { + batch(session, Integer.parseInt(msg.substring("batch:".length()))); + } else if (msg.startsWith("threads:")) { + threads(session, + Integer.parseInt(msg.substring("threads:".length()))); + } else if (msg.startsWith("handler:")) { + remote.sendText(msg, new Reporter(session)); + } else if (msg.startsWith("object:")) { + /* + * An Integer takes sendObject()'s primitive path and is echoed as + * its decimal string; anything else is sent as a String, which + * has no encoder, so the future must complete with a failure. + * That failure is this fork's behaviour (Util.isPrimitive() does + * not count String), not something the specification requires: + * the case is here to show a failed future completes. + */ + String arg = msg.substring("object:".length()); + Object obj; + try { + obj = Integer.valueOf(arg); + } catch (NumberFormatException e) { + obj = arg; + } + report(session, waitFor(remote.sendObject(obj))); + } else { + report(session, waitFor(remote.sendText(msg))); + } + } + + @OnMessage + public void echoBinaryMessage(Session session, ByteBuffer bb) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + if (bb.remaining() > 3 && bb.get(bb.position()) == 'o') { + offset(session, bb); + } else if (bb.remaining() > 0 && bb.get(bb.position()) == 'h') { + remote.sendBinary(bb, new Reporter(session)); + } else { + report(session, waitFor(remote.sendBinary(bb))); + } + } + + private static void batch(Session session, int n) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + try { + remote.setBatchingAllowed(true); + + for (int i = 0; i < n; i++) { + String status = waitFor(remote.sendText("b" + i)); + if (!status.equals("future-done")) { + report(session, "batch-fail: " + status); + return; + } + } + + remote.setBatchingAllowed(false); + } catch (Exception e) { + report(session, "batch-fail: " + e); + return; + } + + report(session, "batch-flushed"); + } + + /* + * With batching allowed, sends the message from its fourth byte on, as a + * buffer whose position() is not 0, then a text frame "after", flushes, + * and reports "offset-flushed". The header of the first frame has to + * give its real length, or the second one is read as part of it. + */ + private static void offset(Session session, ByteBuffer bb) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + try { + remote.setBatchingAllowed(true); + + ByteBuffer copy = ByteBuffer.allocate(bb.remaining()); + copy.put(bb); + copy.flip(); + copy.position(3); + + String status = waitFor(remote.sendBinary(copy)); + if (status.equals("future-done")) { + status = waitFor(remote.sendText("after")); + } + if (!status.equals("future-done")) { + report(session, "offset-fail: " + status); + return; + } + + remote.setBatchingAllowed(false); + } catch (Exception e) { + report(session, "offset-fail: " + e); + return; + } + + report(session, "offset-flushed"); + } + + /* + * Echoes msg with a SendHandler whose onResult() waits on a send of its + * own, "nested-b", then batches "nested-c" and flushes it, and reports + * "nested-done", or the first failure. Both waits need the nested + * send's completion to run before onResult() returns. + */ + private static void nested(Session session, String msg) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + remote.sendText(msg, result -> { + String status = waitFor(remote.sendText("nested-b")); + if (!status.equals("future-done")) { + report(session, "nested-fail: " + status); + return; + } + + try { + remote.setBatchingAllowed(true); + status = waitFor(remote.sendText("nested-c")); + remote.setBatchingAllowed(false); + } catch (Exception e) { + report(session, "nested-fail: " + e); + return; + } + + report(session, status.equals("future-done") + ? "nested-done" : "nested-fail: " + status); + }); + } + + private static void threads(Session session, int n) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + List threads = new ArrayList<>(); + String[] status = new String[n]; + + for (int i = 0; i < n; i++) { + final int k = i; + Thread t = new Thread(() -> { + /* + * A second send on the endpoint while the first is in flight + * is refused by the specification, so retry until this one + * gets its turn; what is under test is that every accepted + * send completes. + */ + for (;;) { + try { + status[k] = waitFor(remote.sendText("t" + k)); + return; + } catch (IllegalStateException e) { + Thread.yield(); + } + } + }); + threads.add(t); + t.start(); + } + + for (int i = 0; i < n; i++) { + try { + threads.get(i).join(10000); + } catch (InterruptedException e) { + report(session, "threads-fail: " + e); + return; + } + if (!"future-done".equals(status[i])) { + report(session, "threads-fail: " + i + " " + status[i]); + return; + } + } + + report(session, "threads-done"); + } + + private static String waitFor(Future future) { + try { + future.get(5, TimeUnit.SECONDS); + } catch (java.util.concurrent.TimeoutException e) { + return "future-timeout"; + } catch (Exception e) { + return "future-fail: " + e; + } + return "future-done"; + } + + private static void report(Session session, String status) { + try { + session.getBasicRemote().sendText(status); + } catch (IOException e) { + // Ignore + } + } + + private static class Reporter implements SendHandler { + private final Session session; + + Reporter(Session session) { + this.session = session; + } + + @Override + public void onResult(SendResult result) { + if (result.isOK()) { + report(session, "handler-ok"); + } else { + report(session, "handler-fail: " + result.getException()); + } + } + } +} diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index a70bf999c..b54ab9403 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -1433,3 +1433,217 @@ def test_java_websockets_binary_message_too_big_reason(): assert 'binary' in frame['reason'], 'reason names the binary message' sock.close() + + +# async remote (issue #434) + + +def test_java_websockets_async_text(): + # The asynchronous remote used to send nothing and never call the + # SendHandler, so the future from sendText(String) never completed. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + # SendHandler form + + ws.frame_write(sock, ws.OP_TEXT, 'handler:blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler:blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + # Future form + + ws.frame_write(sock, ws.OP_TEXT, 'blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'blah') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + # sendObject(Object) with an Integer, which is sent as text + + ws.frame_write(sock, ws.OP_TEXT, 'object:123') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, '123') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + # sendObject(Object) with a String, which has no encoder: the future + # completes with the failure instead of hanging + + ws.frame_write(sock, ws.OP_TEXT, 'object:blah') + frame = ws.frame_read(sock) + assert frame['data'].startswith(b'future-fail: '), 'failed future' + assert b'EncodeException' in frame['data'], 'failed future reason' + + # the session is usable for a further message after each of them + + ws.frame_write(sock, ws.OP_TEXT, 'handler:again') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler:again') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + close_connection(sock) + + +def test_java_websockets_async_binary(): + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + # SendHandler form + + ws.frame_write(sock, ws.OP_BINARY, b'h\x00\x01\x02\xff') + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, b'h\x00\x01\x02\xff') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + # Future form + + ws.frame_write(sock, ws.OP_BINARY, b'\x00\x01\x02\xff') + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, b'\x00\x01\x02\xff') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + close_connection(sock) + + +def test_java_websockets_async_large(): + # A message larger than the module's 8 KiB buffers: the text one still + # leaves in 8 KiB frames on the asynchronous path, the binary one whole. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + payload = '*' * 65536 + + ws.frame_write(sock, ws.OP_TEXT, payload) + check_frame(ws.message_read(sock), True, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + payload = b'*' * 65536 + + ws.frame_write(sock, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + payload = b'h' + b'*' * 65535 + + ws.frame_write(sock, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + close_connection(sock) + + +def test_java_websockets_async_batch(): + # Batched messages are held until the batch is flushed. flushBatch() + # used to hand the request a null payload and throw, so disallowing + # batching failed and the held messages were never sent. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, 'batch:3') + + for i in range(3): + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, f'b{i}') + + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'batch-flushed') + + # the session is usable after the flush + + ws.frame_write(sock, ws.OP_TEXT, 'handler:again') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler:again') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'handler-ok') + + close_connection(sock) + + +def test_java_websockets_async_batch_offset(): + # A batched buffer whose position() is not 0: its frame header must give + # the bytes actually sent, or the frame after it is read as its payload. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + payload = b'o\x00\x01\x02\x03\x04\x05' + + ws.frame_write(sock, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, payload[3:]) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'after') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'offset-flushed') + + close_connection(sock) + + +def test_java_websockets_async_nested(): + # A SendHandler that waits on a send of its own, and one that flushes a + # batch: the nested completion has to run before the handler returns. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, 'nested:a') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'nested:a') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'nested-b') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'nested-c') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'nested-done') + + close_connection(sock) + + +def test_java_websockets_async_threads(): + # Sends from several threads at once: every one of them is echoed and + # every future completes. A completion is queued behind a running one + # only on the same thread; a queue shared by the endpoint could take a + # second thread's completion after the first had stopped draining it. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + n = 8 + + for _ in range(10): + ws.frame_write(sock, ws.OP_TEXT, f'threads:{n}') + + echoes = set() + for _ in range(n): + frame = ws.frame_read(sock) + assert frame['opcode'] == ws.OP_TEXT, 'echo opcode' + echoes.add(frame['data'].decode()) + + assert echoes == {f't{i}' for i in range(n)}, 'every send echoed' + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'threads-done') + + close_connection(sock) + + +def test_java_websockets_async_16m(is_unsafe): + if not is_unsafe: + pytest.skip('unsafe, long run') + + # The asynchronous text path sends a message 8 KiB at a time and starts + # each part from the completion of the previous one; a 16 MiB message is + # 2048 parts, which must not nest on the stack. + client.load('websockets_async') + + assert 'success' in client.conf( + { + 'http': { + 'websocket': { + 'max_frame_size': 33554432, + 'keepalive_interval': 0, + } + } + }, + 'settings', + ), 'increase max_frame_size and keepalive_interval' + + _, sock, _ = ws.upgrade() + + payload = '*' * 16 * 2**20 + + ws.frame_write(sock, ws.OP_TEXT, payload) + check_frame(ws.message_read(sock), True, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + payload = b'*' * 16 * 2**20 + + ws.frame_write(sock, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + close_connection(sock) From ddde07cbe7303187897a4a789be40284437e3590 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 22:58:01 +0000 Subject: [PATCH 257/391] java: send a WebSocket frame from where its buffer starts Request.sendWsFrame() handed a heap buffer to the native call as payload.array() with payload.position() as the offset, ignoring arrayOffset(). A buffer whose content does not start at the first byte of its array, such as ByteBuffer.wrap(a, off, n).slice(), was sent from the wrong place in the array, and a read-only heap buffer threw ReadOnlyBufferException from array() instead of being sent. The blocking remote passes the user's buffer through, and so does the asynchronous one since #434. The direct-buffer branch was not affected. An array-backed buffer is now sent from arrayOffset() + position(). A heap buffer without an accessible array has its remaining bytes copied through a duplicate, so the caller's position is left as it was, and the copy is sent. No other JNI send path takes a ByteBuffer: the response body writes take a byte[] with an explicit offset. Tests: a websockets_buffers app sends a sliced buffer and a read-only one through getBasicRemote().sendBinary() and through the Future form of getAsyncRemote().sendBinary(), and the client checks it receives exactly the intended bytes. Both tests fail on master. Closes #486. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 9 ++++++ docs/changes.xml | 12 ++++++++ src/java/nginx/unit/Request.java | 16 +++++++++-- test/java/websockets_buffers/app.java | 41 +++++++++++++++++++++++++++ test/test_java_websockets.py | 24 ++++++++++++++++ 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 test/java/websockets_buffers/app.java diff --git a/CHANGES b/CHANGES index 106de7df4..da49ba80a 100644 --- a/CHANGES +++ b/CHANGES @@ -428,6 +428,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 buffer whose position was not 0 had a header that overstated its length and corrupted the frames after it. + *) Bugfix: the java module sent the wrong bytes, or nothing, for a + WebSocket message from a heap ByteBuffer that does not start at its + array's first byte. The payload was read from the backing array at + position(), ignoring arrayOffset(), so a buffer from + ByteBuffer.wrap(a, off, n).slice() sent bytes from the start of the + array instead of its own; and a read-only heap buffer threw + ReadOnlyBufferException instead of being sent. Both the blocking and + the asynchronous remote were affected (#486). + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index 1fcb94494..eeebc6087 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -562,6 +562,18 @@ header that overstated its length and corrupted the frames after it. + + +the java module sent the wrong bytes, or nothing, for a WebSocket message +from a heap ByteBuffer that does not start at its array's first byte. The +payload was read from the backing array at position(), ignoring arrayOffset(), +so a buffer from ByteBuffer.wrap(a, off, n).slice() sent bytes from the start +of the array instead of its own; and a read-only heap buffer threw +ReadOnlyBufferException instead of being sent. Both the blocking and the +asynchronous remote were affected (#486). + + + diff --git a/src/java/nginx/unit/Request.java b/src/java/nginx/unit/Request.java index 335d79805..0352957ef 100644 --- a/src/java/nginx/unit/Request.java +++ b/src/java/nginx/unit/Request.java @@ -1310,8 +1310,20 @@ public void sendWsFrame(ByteBuffer payload, byte opCode, boolean last, sendWsFrame(req_info_ptr, payload, payload.position(), payload.limit() - payload.position(), opCode, last); } else { - sendWsFrame(req_info_ptr, payload.array(), payload.position(), - payload.limit() - payload.position(), opCode, last); + if (!payload.hasArray()) { + /* + * A read-only heap buffer hides its array, so each frame + * sent from one costs a copy of its remaining bytes. + */ + ByteBuffer copy = ByteBuffer.allocate(payload.remaining()); + copy.put(payload.duplicate()); + copy.flip(); + payload = copy; + } + + sendWsFrame(req_info_ptr, payload.array(), + payload.arrayOffset() + payload.position(), + payload.remaining(), opCode, last); } } diff --git a/test/java/websockets_buffers/app.java b/test/java/websockets_buffers/app.java new file mode 100644 index 000000000..a5229b13e --- /dev/null +++ b/test/java/websockets_buffers/app.java @@ -0,0 +1,41 @@ +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import javax.websocket.OnMessage; +import javax.websocket.Session; +import javax.websocket.server.ServerEndpoint; + +/* + * Sends the bytes "456789ab" from a heap buffer that does not start at its + * array's first byte (issue #486): a slice for ":slice", a read-only + * buffer for ":readonly", through the async remote when is + * "async" and the blocking one otherwise. Then reports "done" or + * "fail: ...". + */ +@ServerEndpoint("/") +public class app { + + private static final byte[] DATA = + "0123456789abcdef".getBytes(StandardCharsets.US_ASCII); + + @OnMessage + public void onMessage(Session session, String msg) throws IOException { + ByteBuffer bb = ByteBuffer.wrap(DATA, 4, 8); + bb = msg.endsWith(":slice") ? bb.slice() : bb.asReadOnlyBuffer(); + + try { + if (msg.startsWith("async:")) { + session.getAsyncRemote().sendBinary(bb).get(5, TimeUnit.SECONDS); + } else { + session.getBasicRemote().sendBinary(bb); + } + } catch (Exception e) { + session.getBasicRemote().sendText("fail: " + e); + return; + } + + session.getBasicRemote().sendText("done"); + } +} diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index b54ab9403..723acb750 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -1647,3 +1647,27 @@ def test_java_websockets_async_16m(is_unsafe): check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') close_connection(sock) + + +def check_buffers(remote): + # Heap buffers whose bytes do not start at their array's first one: a + # slice with a non-zero arrayOffset() used to be sent from the start of + # the array, and a read-only buffer failed with ReadOnlyBufferException. + client.load('websockets_buffers') + + _, sock, _ = ws.upgrade() + + for kind in ('slice', 'readonly'): + ws.frame_write(sock, ws.OP_TEXT, f'{remote}:{kind}') + check_frame(ws.frame_read(sock), True, ws.OP_BINARY, b'456789ab') + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'done') + + close_connection(sock) + + +def test_java_websockets_buffers(): + check_buffers('basic') + + +def test_java_websockets_async_buffers(): + check_buffers('async') From 3e4b343a0a393b8020ed40d1390f5e37f060780c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 22:56:45 +0000 Subject: [PATCH 258/391] fix(buf): do not memcpy() from an empty string nxt_buf_cpy() called nxt_memcpy(), which is a plain memcpy() wrapper. memcpy() declares its source argument non-null, even for a zero-length copy. nxt_buf_cpystr() can be called with an empty nxt_str_t, whose start pointer is NULL. This is undefined behaviour and UBSan reports it: src/nxt_buf.h:250:5: runtime error: null pointer passed as argument 2, which is declared to never be null This happens in nxt_router_start_app_process_handler(), when an application has no "conf" string to pass to the prototype process. The fix switches nxt_buf_cpy() to nxt_cpymem(), which already has a zero-length guard, written for the same reason, in nxt_string.h. No other memcpy() call in nxt_buf.h has this issue. Added src/test/nxt_buf_test.c: it calls nxt_buf_cpystr() with an empty nxt_str_t. Without the fix, a UBSan build fails on this test with the same runtime error, right after "nxt_string helpers test passed". Co-Authored-By: Claude Opus 5.5 --- auto/sources | 1 + src/nxt_buf.h | 4 ++-- src/test/nxt_buf_test.c | 46 +++++++++++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 ++++ src/test/nxt_tests.h | 1 + 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 src/test/nxt_buf_test.c diff --git a/auto/sources b/auto/sources index 9682b9598..9d727eba1 100644 --- a/auto/sources +++ b/auto/sources @@ -165,6 +165,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_strverscmp_test.c \ src/test/nxt_base64_test.c \ src/test/nxt_string_test.c \ + src/test/nxt_buf_test.c \ src/test/nxt_http_chunk_parse_test.c \ src/test/nxt_conf_parse_test.c \ src/test/nxt_port_fail_test.c \ diff --git a/src/nxt_buf.h b/src/nxt_buf.h index 2f2cbaa68..9ba911599 100644 --- a/src/nxt_buf.h +++ b/src/nxt_buf.h @@ -247,8 +247,8 @@ NXT_EXPORT size_t nxt_buf_chain_length(nxt_buf_t *b); nxt_inline nxt_buf_t * nxt_buf_cpy(nxt_buf_t *b, const void *src, size_t length) { - nxt_memcpy(b->mem.free, src, length); - b->mem.free += length; + /* nxt_cpymem() skips a zero-length copy, whose "src" may be NULL. */ + b->mem.free = nxt_cpymem(b->mem.free, src, length); return b; } diff --git a/src/test/nxt_buf_test.c b/src/test/nxt_buf_test.c new file mode 100644 index 000000000..f9e3159d9 --- /dev/null +++ b/src/test/nxt_buf_test.c @@ -0,0 +1,46 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +#include +#include "nxt_tests.h" + + +/* An empty nxt_str_t has a NULL start: memcpy() must not be given it. */ + +nxt_int_t +nxt_buf_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_mp_t *mp; + nxt_buf_t *b; + static nxt_str_t empty = nxt_null_string; + + mp = nxt_mp_create(1024, 128, 512, 16); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + + b = nxt_buf_mem_alloc(mp, 1, 0); + + if (b != NULL) { + nxt_buf_cpystr(b, &empty); + + if (nxt_buf_mem_used_size(&b->mem) == 0) { + ret = NXT_OK; + } + } + + nxt_mp_destroy(mp); + + if (ret != NXT_OK) { + nxt_log_alert(thr->log, "nxt_buf test failed"); + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nxt_buf test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index da6d7d84d..c55169a4d 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -179,6 +179,10 @@ main(int argc, char **argv) return 1; } + if (nxt_buf_test(thr) != NXT_OK) { + return 1; + } + if (nxt_http_chunk_parse_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index c6dcafeec..e60d6c4ed 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -66,6 +66,7 @@ nxt_int_t nxt_http_parse_test(nxt_thread_t *thr); nxt_int_t nxt_strverscmp_test(nxt_thread_t *thr); nxt_int_t nxt_base64_test(nxt_thread_t *thr); nxt_int_t nxt_string_test(nxt_thread_t *thr); +nxt_int_t nxt_buf_test(nxt_thread_t *thr); nxt_int_t nxt_http_chunk_parse_test(nxt_thread_t *thr); nxt_int_t nxt_conf_json_depth_test(nxt_thread_t *thr); nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); From d22974c3f3bf9a0879ad0ed04a9483dc445d5399 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 01:42:44 +0200 Subject: [PATCH 259/391] java: send an asynchronous WebSocket text message as one frame #433 sized the encode buffer only on the blocking path. Since #478 the asynchronous remote sends, and sendStringByCompletion() still used the fixed 8 KiB encoderBuffer, so a 64 KiB text message from getAsyncRemote() left as eight frames. Both paths now take their buffer from encodeBuffer(). The one-frame check now runs on both remotes in the default suite. The asynchronous one fails on master with a first frame of 8192 bytes, fin clear. async_16m pins the inclusive 16 MiB cap, then lowers the cap to 8 KiB to keep the 2048-part completion chain covered. The #430 changelog entry now covers both remotes and corrects the Writer case to up to four frames. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 16 ++--- docs/changes.xml | 14 +++-- .../websocket/WsRemoteEndpointImplBase.java | 38 ++++++------ test/test_java_websockets.py | 62 +++++++++++++++++-- 4 files changed, 93 insertions(+), 37 deletions(-) diff --git a/CHANGES b/CHANGES index da49ba80a..db122d6f9 100644 --- a/CHANGES +++ b/CHANGES @@ -406,13 +406,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 to. A 64 KiB message arrived as eight frames and a 16 MiB one as about two thousand, each carrying its own frame header and costing its own message to the router, while every other language module sends a - complete message as a single frame. A complete text message is now - encoded into a buffer of its own and sent as one frame; a message - above "nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 MiB by default, - still fragments. A Writer from getSendWriter() still sends each chunk - it flushes as a frame of its own, but that chunk is now one frame - instead of up to three. This does not make the module honour - "max_frame_size", which remains an inbound limit only. + complete message as a single frame. A complete text message, from + getBasicRemote() or getAsyncRemote(), is now encoded into a buffer of + its own and sent as one frame; a message above + "nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 MiB by default, still + fragments. A Writer from getSendWriter() still sends each chunk it + flushes as a frame of its own, but that chunk is now one frame instead + of up to four. An OutputStream from getSendStream() still sends each + 8 KiB it fills as a frame of its own. This does not make the module + honour "max_frame_size", which remains an inbound limit only. *) Bugfix: the java module sent nothing through the asynchronous WebSocket remote. Every send from getAsyncRemote() -- sendText(), diff --git a/docs/changes.xml b/docs/changes.xml index eeebc6087..5ffebb77c 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -537,12 +537,14 @@ into 8 KiB frames, whatever "max_frame_size" was set to. A 64 KiB message arrived as eight frames and a 16 MiB one as about two thousand, each carrying its own frame header and costing its own message to the router, while every other language module sends a complete message as a single frame. A complete -text message is now encoded into a buffer of its own and sent as one frame; a -message above "nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 MiB by default, -still fragments. A Writer from getSendWriter() still sends each chunk it -flushes as a frame of its own, but that chunk is now one frame instead of up to -three. This does not make the module honour "max_frame_size", which remains an -inbound limit only. +text message, from getBasicRemote() or getAsyncRemote(), is now encoded into a +buffer of its own and sent as one frame; a message above +"nginx.unit.websocket.MAX_SEND_BUFFER_SIZE", 16 MiB by default, still +fragments. A Writer from getSendWriter() still sends each chunk it flushes as +a frame of its own, but that chunk is now one frame instead of up to four. An +OutputStream from getSendStream() still sends each 8 KiB it fills as a frame of +its own. This does not make the module honour "max_frame_size", which remains +an inbound limit only. diff --git a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java index 6d4ad509a..910ec63f9 100644 --- a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java +++ b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java @@ -231,8 +231,9 @@ public void sendStringByCompletion(String text, SendHandler handler) { throw new IllegalArgumentException(sm.getString("wsRemoteEndpoint.nullHandler")); } stateMachine.textStart(); + CharBuffer part = CharBuffer.wrap(text); TextMessageSendHandler tmsh = new TextMessageSendHandler(handler, - CharBuffer.wrap(text), true, encoder, encoderBuffer, this); + part, true, encoder, encodeBuffer(part), this); tmsh.write(); // TextMessageSendHandler will update stateMachine when it completes } @@ -313,28 +314,29 @@ private static long utf8Length(CharBuffer part, long cap) { } - void sendMessageBlock(CharBuffer part, boolean last) throws IOException { - long timeoutExpiry = getTimeoutExpiry(); - boolean isDone = false; - - /* - * Encode the whole message into one buffer where it fits under the - * cap, so that it leaves as a single frame. Above the cap, and for - * the messages that already fit, encoderBuffer is used as before; the - * loop is unchanged, so an oversized message still fragments exactly - * as it did. - * - * The buffer is on the heap and not direct, to keep the large message - * on the same JNI path as every message that fits in encoderBuffer, - * which is itself a heap buffer. - */ - ByteBuffer buffer = encoderBuffer; + /* + * A message that fits under the cap gets a buffer of its own, so it leaves + * as one frame: Utf8Encoder returns UNDERFLOW when input and output run + * out together. Any other message uses encoderBuffer and fragments as + * before. The buffer is on the heap, like encoderBuffer, so both take the + * same JNI path. + */ + private ByteBuffer encodeBuffer(CharBuffer part) { long needed = utf8Length(part, Constants.MAX_SEND_BUFFER_SIZE); if (needed != OVER_CAP && needed > encoderBuffer.capacity()) { - buffer = ByteBuffer.allocate((int) needed); + return ByteBuffer.allocate((int) needed); } + return encoderBuffer; + } + + + void sendMessageBlock(CharBuffer part, boolean last) throws IOException { + long timeoutExpiry = getTimeoutExpiry(); + boolean isDone = false; + ByteBuffer buffer = encodeBuffer(part); + while (!isDone) { buffer.clear(); CoderResult cr = encoder.encode(part, buffer, true); diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 723acb750..598cd88a8 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -121,6 +121,29 @@ def test_java_websockets_fragmentation(): sock.close() +def check_text_one_frame(app, *after): + # A text message larger than the module's 8 KiB encoder buffer is sent + # as one frame. frame_read(), not message_read(), which would join the + # fragments and hide them. + client.load(app) + + _, sock, _ = ws.upgrade() + + payload = '*' * 64 * 2**10 + + ws.frame_write(sock, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, payload) + + for text in after: + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, text) + + close_connection(sock) + + +def test_java_websockets_text_one_frame(): + check_text_one_frame('websockets_mirror') + + def test_java_websockets_frame_fragmentation_invalid(): client.load('websockets_mirror') @@ -1501,8 +1524,9 @@ def test_java_websockets_async_binary(): def test_java_websockets_async_large(): - # A message larger than the module's 8 KiB buffers: the text one still - # leaves in 8 KiB frames on the asynchronous path, the binary one whole. + # A message larger than the module's 8 KiB buffers is sent whole through + # the asynchronous remote. test_java_websockets_async_text_one_frame + # checks the text message on the wire, frame by frame. client.load('websockets_async') _, sock, _ = ws.upgrade() @@ -1528,6 +1552,10 @@ def test_java_websockets_async_large(): close_connection(sock) +def test_java_websockets_async_text_one_frame(): + check_text_one_frame('websockets_async', 'future-done') + + def test_java_websockets_async_batch(): # Batched messages are held until the batch is flushed. flushBatch() # used to hand the request a null payload and throw, so disallowing @@ -1615,9 +1643,10 @@ def test_java_websockets_async_16m(is_unsafe): if not is_unsafe: pytest.skip('unsafe, long run') - # The asynchronous text path sends a message 8 KiB at a time and starts - # each part from the completion of the previous one; a 16 MiB message is - # 2048 parts, which must not nest on the stack. + # 16 MiB is exactly the default send cap, so it is one frame. Above the + # cap a message is sent 8 KiB at a time, each part started from the + # previous one's completion; lowering the cap to 8 KiB makes 2048 parts, + # which must not nest on the stack. client.load('websockets_async') assert 'success' in client.conf( @@ -1637,7 +1666,7 @@ def test_java_websockets_async_16m(is_unsafe): payload = '*' * 16 * 2**20 ws.frame_write(sock, ws.OP_TEXT, payload) - check_frame(ws.message_read(sock), True, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, payload) check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') payload = b'*' * 16 * 2**20 @@ -1648,6 +1677,27 @@ def test_java_websockets_async_16m(is_unsafe): close_connection(sock) + assert 'success' in client.conf( + ['-Dnginx.unit.websocket.MAX_SEND_BUFFER_SIZE=8192'], + 'applications/websockets_async/options', + ), 'lower the send buffer cap' + + _, sock, _ = ws.upgrade() + + payload = '*' * 16 * 2**20 + + ws.frame_write(sock, ws.OP_TEXT, payload) + + sizes = [] + frame = {'fin': False} + while not frame['fin']: + frame = ws.frame_read(sock) + sizes.append(len(frame['data'])) + + assert sizes == [8192] * 2048, 'parts above the cap' + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + close_connection(sock) def check_buffers(remote): # Heap buffers whose bytes do not start at their array's first one: a From e9ff6e67f61e624a64c7411e130bd2d4e370800a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 23:53:39 +0000 Subject: [PATCH 260/391] tests: cover the async one-frame text path more widely, and name its cost Review follow-ups: - test_java_websockets_async_text_one_frame_forms: one frame through the SendHandler form as well as the Future one, for characters of every UTF-8 width (1, 2, 3 and 4 bytes, 40 KiB) so that utf8Length() sizes the buffer for each, and with batching on, where the frame reaches the wire through the 8 KiB output buffer in parts. It fails with master's WsRemoteEndpointImplBase.java: the first frame is 8192 bytes. - test_java_websockets_async_text_above_cap: with the cap lowered to 8 KiB, a 32 KiB text message is four 8 KiB frames, fin on the last. This puts the fragmenting branch in the default suite; before, only the --unsafe 16 MiB case reached it. - encodeBuffer()'s comment names what both paths pay for one frame: a full utf8Length() pass over the message before the first byte is sent, and a buffer of the message's encoded length, up to 16 MiB by default. sendObject() above 8 KiB stays untested: the only objects this fork sends as text without an encoder are the boxed primitives, which never reach 8 KiB. Co-Authored-By: Claude Opus 5.5 --- .../websocket/WsRemoteEndpointImplBase.java | 4 ++ test/java/websockets_async/app.java | 24 +++++++- test/test_java_websockets.py | 56 +++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java index 910ec63f9..3e2fd9a79 100644 --- a/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java +++ b/src/java/nginx/unit/websocket/WsRemoteEndpointImplBase.java @@ -320,6 +320,10 @@ private static long utf8Length(CharBuffer part, long cap) { * out together. Any other message uses encoderBuffer and fragments as * before. The buffer is on the heap, like encoderBuffer, so both take the * same JNI path. + * + * The price, on the blocking and the asynchronous path alike: a full + * utf8Length() pass before the first byte is sent, and a buffer of the + * message's encoded length, up to 16 MiB by default. */ private ByteBuffer encodeBuffer(CharBuffer part) { long needed = utf8Length(part, Constants.MAX_SEND_BUFFER_SIZE); diff --git a/test/java/websockets_async/app.java b/test/java/websockets_async/app.java index a3267ed9c..aa5c33520 100644 --- a/test/java/websockets_async/app.java +++ b/test/java/websockets_async/app.java @@ -38,6 +38,7 @@ * completed, or the first failure or timeout. A binary message whose first * byte is 'o' is sent batched from its fourth byte on; see offset(). * "nested:..." waits on sends from inside a SendHandler; see nested(). + * "batchtext:..." is echoed batched and then flushed; see batchText(). */ @ServerEndpoint("/") public class app { @@ -52,7 +53,9 @@ public void onOpen(Session session) { public void echoTextMessage(Session session, String msg) { RemoteEndpoint.Async remote = session.getAsyncRemote(); - if (msg.startsWith("nested:")) { + if (msg.startsWith("batchtext:")) { + batchText(session, msg); + } else if (msg.startsWith("nested:")) { nested(session, msg); } else if (msg.startsWith("batch:")) { batch(session, Integer.parseInt(msg.substring("batch:".length()))); @@ -184,6 +187,25 @@ private static void nested(Session session, String msg) { }); } + /* + * Echoes msg with batching allowed, then disallows it, which flushes the + * echo, and reports "batchtext-flushed" or "batchtext-fail: ...". + */ + private static void batchText(Session session, String msg) { + RemoteEndpoint.Async remote = session.getAsyncRemote(); + + try { + remote.setBatchingAllowed(true); + String status = waitFor(remote.sendText(msg)); + remote.setBatchingAllowed(false); + + report(session, status.equals("future-done") + ? "batchtext-flushed" : "batchtext-fail: " + status); + } catch (Exception e) { + report(session, "batchtext-fail: " + e); + } + } + private static void threads(Session session, int n) { RemoteEndpoint.Async remote = session.getAsyncRemote(); List threads = new ArrayList<>(); diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 598cd88a8..50b55f802 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -1556,6 +1556,62 @@ def test_java_websockets_async_text_one_frame(): check_text_one_frame('websockets_async', 'future-done') +def test_java_websockets_async_text_one_frame_forms(): + # One frame also through the SendHandler form, for characters of every + # UTF-8 width, and with batching on, where the frame reaches the wire + # through the 8 KiB output buffer in parts. + client.load('websockets_async') + + _, sock, _ = ws.upgrade() + + wide = 'a\u00e9\u20ac\U0001f600' * 4096 # 1, 2, 3 and 4 bytes, 40 KiB + + for payload, report in ( + ('handler:' + '*' * 64 * 2**10, 'handler-ok'), + (wide, 'future-done'), + ('handler:' + wide, 'handler-ok'), + ('batchtext:' + '*' * 64 * 2**10, 'batchtext-flushed'), + ): + ws.frame_write(sock, ws.OP_TEXT, payload) + + frame = ws.frame_read(sock) + assert frame['fin'], f'one frame, got {len(frame["data"])} bytes' + check_frame(frame, True, ws.OP_TEXT, payload) + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, report) + + close_connection(sock) + + +def test_java_websockets_async_text_above_cap(): + # Above nginx.unit.websocket.MAX_SEND_BUFFER_SIZE a text message still + # fragments, 8 KiB a frame; with the cap lowered to 8 KiB a 32 KiB + # message is four frames. + client.load('websockets_async') + + assert 'success' in client.conf( + ['-Dnginx.unit.websocket.MAX_SEND_BUFFER_SIZE=8192'], + 'applications/websockets_async/options', + ), 'lower the send buffer cap' + + _, sock, _ = ws.upgrade() + + payload = '*' * 32 * 2**10 + + ws.frame_write(sock, ws.OP_TEXT, payload) + + frames = [ws.frame_read(sock) for _ in range(4)] + assert [(f['opcode'], f['fin'], len(f['data'])) for f in frames] == [ + (ws.OP_TEXT, False, 8192), + (ws.OP_CONT, False, 8192), + (ws.OP_CONT, False, 8192), + (ws.OP_CONT, True, 8192), + ], 'four parts' + assert b''.join(f['data'] for f in frames).decode() == payload, 'payload' + check_frame(ws.frame_read(sock), True, ws.OP_TEXT, 'future-done') + + close_connection(sock) + + def test_java_websockets_async_batch(): # Batched messages are held until the batch is flushed. flushBatch() # used to hand the request a null payload and throw, so disallowing From 3ac8becd241d24a3a5dca15b449a023827df538c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 01:10:19 +0000 Subject: [PATCH 261/391] fix(conf): store mapped config values through memcpy, not a typed union nxt_conf_map_object() stored each value through a pointer to a union whose widest members need 8-byte alignment, cast from an arbitrary offset into the caller's struct. Most destination fields are less aligned than that, so every store was undefined behaviour: UBSan reported "member access within misaligned address" under test/test_settings.py, and strict-alignment targets can fault. Build each value in a local union and nxt_memcpy() it to the destination. The small copies still compile to plain moves. A value of the wrong JSON type still leaves the field unchanged. The two inner "default" arms "continue" after nxt_unreachable(), so "len" is never used unset where nxt_unreachable() expands to nothing. freeunitorg/freeunit#188. Co-Authored-By: Claude Opus 5.5 --- src/nxt_conf.c | 66 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/nxt_conf.c b/src/nxt_conf.c index ec4ed7b07..3a66462f0 100644 --- a/src/nxt_conf.c +++ b/src/nxt_conf.c @@ -607,10 +607,15 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, const nxt_conf_map_t *map, nxt_uint_t n, void *data) { double num; + size_t len; nxt_str_t str, *s; nxt_uint_t i; nxt_conf_value_t *v; + /* + * A mapped field can have less alignment than its type needs. + * Build the value here and copy it to the field with nxt_memcpy(). + */ union { uint8_t ui8; int32_t i32; @@ -623,7 +628,7 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, nxt_str_t str; char *cstrz; void *v; - } *ptr; + } val; for (i = 0; i < n; i++) { @@ -633,16 +638,16 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, continue; } - ptr = nxt_pointer_to(data, map[i].offset); - switch (map[i].type) { case NXT_CONF_MAP_INT8: - if (v->type == NXT_CONF_VALUE_BOOLEAN) { - ptr->ui8 = v->u.boolean; + if (v->type != NXT_CONF_VALUE_BOOLEAN) { + continue; } + val.ui8 = v->u.boolean; + len = sizeof(val.ui8); break; case NXT_CONF_MAP_INT32: @@ -653,7 +658,7 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, case NXT_CONF_MAP_MSEC: if (v->type != NXT_CONF_VALUE_INTEGER) { - break; + continue; } num = nxt_strtod(v->u.number, NULL); @@ -661,41 +666,50 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, switch (map[i].type) { case NXT_CONF_MAP_INT32: - ptr->i32 = num; + val.i32 = num; + len = sizeof(val.i32); break; case NXT_CONF_MAP_INT64: - ptr->i64 = num; + val.i64 = num; + len = sizeof(val.i64); break; case NXT_CONF_MAP_INT: - ptr->i = num; + val.i = num; + len = sizeof(val.i); break; case NXT_CONF_MAP_SIZE: - ptr->size = num; + val.size = num; + len = sizeof(val.size); break; case NXT_CONF_MAP_OFF: - ptr->off = num; + val.off = num; + len = sizeof(val.off); break; case NXT_CONF_MAP_MSEC: - ptr->msec = (nxt_msec_t) num * 1000; + val.msec = (nxt_msec_t) num * 1000; + len = sizeof(val.msec); break; default: nxt_unreachable(); + continue; } break; case NXT_CONF_MAP_DOUBLE: - if (v->type == NXT_CONF_VALUE_NUMBER) { - ptr->dbl = nxt_strtod(v->u.number, NULL); + if (v->type != NXT_CONF_VALUE_NUMBER) { + continue; } + val.dbl = nxt_strtod(v->u.number, NULL); + len = sizeof(val.dbl); break; case NXT_CONF_MAP_STR: @@ -705,7 +719,7 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, if (v->type != NXT_CONF_VALUE_SHORT_STRING && v->type != NXT_CONF_VALUE_STRING) { - break; + continue; } nxt_conf_get_string(v, &str); @@ -713,41 +727,49 @@ nxt_conf_map_object(nxt_mp_t *mp, const nxt_conf_value_t *value, switch (map[i].type) { case NXT_CONF_MAP_STR: - ptr->str = str; + val.str = str; + len = sizeof(val.str); break; case NXT_CONF_MAP_STR_COPY: - s = nxt_str_dup(mp, &ptr->str, &str); + s = nxt_str_dup(mp, &val.str, &str); if (nxt_slow_path(s == NULL)) { return NXT_ERROR; } + len = sizeof(val.str); break; case NXT_CONF_MAP_CSTRZ: - ptr->cstrz = nxt_str_cstrz(mp, &str); + val.cstrz = nxt_str_cstrz(mp, &str); - if (nxt_slow_path(ptr->cstrz == NULL)) { + if (nxt_slow_path(val.cstrz == NULL)) { return NXT_ERROR; } + len = sizeof(val.cstrz); break; default: nxt_unreachable(); + continue; } break; case NXT_CONF_MAP_PTR: - - ptr->v = v; - + val.v = v; + len = sizeof(val.v); break; + + default: + continue; } + + nxt_memcpy(nxt_pointer_to(data, map[i].offset), &val, len); } return NXT_OK; From 7c2ba8fb838fb433b6922a90acbaedc1cab99a83 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 01:10:19 +0000 Subject: [PATCH 262/391] fix(http): guard nxt_http_route_memcmp() against an empty pattern slice An empty pattern ("") gets one EXACT slice with a NULL start and zero length (nxt_http_route_pattern_slice()). When the request value is also empty, nxt_http_route_memcmp() passed that NULL to memcmp(), which declares its pointers non-null even for length 0. UBSan reported it under test/test_routing.py. An empty slice always matches, so return early, as nxt_strstr_eq() does. The early return also covers the nxt_memcasecmp() path. freeunitorg/freeunit#200. Co-Authored-By: Claude Opus 5.5 --- src/nxt_http_route.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index 5b937dd7d..ce3eb0a84 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -2253,6 +2253,11 @@ nxt_http_route_memcmp(u_char *start, u_char *test, size_t test_length, { nxt_int_t n; + /* A "" pattern slice has a NULL start: no memcmp(NULL, p, 0). */ + if (test_length == 0) { + return 1; + } + if (case_sensitive) { n = memcmp(start, test, test_length); From b7d503e3f1e332235d381e1854d53ff477eee7a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 01:10:19 +0000 Subject: [PATCH 263/391] test(conf): cover nxt_conf_map_object() with a misaligning destination Map a JSON object into a packed struct, so that most fields are misaligned for their type, and check every value. The map covers every NXT_CONF_MAP_* type, and two members of the wrong JSON type, which must keep their sentinel values. On a UBSan build this reports the misaligned store if nxt_conf_map_object() goes back to writing through a typed pointer. freeunitorg/freeunit#188. Co-Authored-By: Claude Opus 5.5 --- src/test/nxt_conf_parse_test.c | 114 +++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 ++ src/test/nxt_tests.h | 1 + 3 files changed, 119 insertions(+) diff --git a/src/test/nxt_conf_parse_test.c b/src/test/nxt_conf_parse_test.c index 1df388edd..e673c75a7 100644 --- a/src/test/nxt_conf_parse_test.c +++ b/src/test/nxt_conf_parse_test.c @@ -147,3 +147,117 @@ nxt_http_route_addr_test(nxt_thread_t *thr) return NXT_OK; } + + + +/* Packed: the fields after "flag" are misaligned, for UBSan to catch. */ + +typedef struct { + uint8_t flag; + int32_t i32; + int64_t i64; + int i; + ssize_t size; + off_t off; + nxt_msec_t msec; + double dbl; + nxt_str_t str; + char *cstrz; + nxt_str_t rstr; + void *ptr; + uint8_t bad8; + int32_t bad32; +} nxt_packed nxt_conf_map_test_t; + + +#define nxt_conf_map_test_field(field, type) \ + { nxt_string(#field), type, offsetof(nxt_conf_map_test_t, field) } + + +nxt_int_t +nxt_conf_map_object_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_conf_value_t *cv; + nxt_conf_map_test_t dst; + + static const nxt_str_t json = nxt_string( + "{\"flag\": true, \"i32\": -123456, \"i64\": -9876543210, \"i\": -7," + " \"size\": 424242, \"off\": 131072, \"msec\": 3, \"dbl\": 1.5," + " \"str\": \"hello\", \"cstrz\": \"world\", \"rstr\": \"raw\"," + " \"ptr\": [1], \"bad8\": 1, \"bad32\": \"x\"}"); + + static const nxt_conf_map_t map[] = { + nxt_conf_map_test_field(flag, NXT_CONF_MAP_INT8), + nxt_conf_map_test_field(i32, NXT_CONF_MAP_INT32), + nxt_conf_map_test_field(i64, NXT_CONF_MAP_INT64), + nxt_conf_map_test_field(i, NXT_CONF_MAP_INT), + nxt_conf_map_test_field(size, NXT_CONF_MAP_SIZE), + nxt_conf_map_test_field(off, NXT_CONF_MAP_OFF), + nxt_conf_map_test_field(msec, NXT_CONF_MAP_MSEC), + nxt_conf_map_test_field(dbl, NXT_CONF_MAP_DOUBLE), + nxt_conf_map_test_field(str, NXT_CONF_MAP_STR_COPY), + nxt_conf_map_test_field(cstrz, NXT_CONF_MAP_CSTRZ), + nxt_conf_map_test_field(rstr, NXT_CONF_MAP_STR), + nxt_conf_map_test_field(ptr, NXT_CONF_MAP_PTR), + /* Wrong JSON types: the fields keep their sentinels. */ + nxt_conf_map_test_field(bad8, NXT_CONF_MAP_INT8), + nxt_conf_map_test_field(bad32, NXT_CONF_MAP_INT32), + }; + + nxt_thread_time_update(thr); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + nxt_memzero(&dst, sizeof(dst)); + dst.bad8 = 0xA5; + dst.bad32 = 0x5A5A5A5A; + + cv = nxt_conf_json_parse(mp, json.start, json.start + json.length, NULL); + + if (cv == NULL + || nxt_conf_map_object(mp, cv, map, nxt_nitems(map), &dst) != NXT_OK) + { + nxt_log_alert(thr->log, "nxt_conf_map_object() test failed"); + goto done; + } + + if (dst.flag != 1 + || dst.i32 != -123456 + || dst.i64 != -9876543210LL + || dst.i != -7 + || dst.size != 424242 + || dst.off != 131072 + || dst.msec != 3000 + || dst.dbl != 1.5 + || dst.str.length != 5 + || memcmp(dst.str.start, "hello", 5) != 0 + || dst.cstrz == NULL + || strcmp(dst.cstrz, "world") != 0 + || dst.rstr.length != 3 + || memcmp(dst.rstr.start, "raw", 3) != 0 + || dst.ptr == NULL + || nxt_conf_type(dst.ptr) != NXT_CONF_ARRAY + || dst.bad8 != 0xA5 + || dst.bad32 != 0x5A5A5A5A) + { + nxt_log_alert(thr->log, "nxt_conf_map_object() mapped wrong values"); + goto done; + } + + ret = NXT_OK; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "nxt_conf_map_object() alignment test passed"); + +done: + + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index c55169a4d..5eab4b6e1 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -195,6 +195,10 @@ main(int argc, char **argv) return 1; } + if (nxt_conf_map_object_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_fail_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index e60d6c4ed..adb6bb37b 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -70,6 +70,7 @@ nxt_int_t nxt_buf_test(nxt_thread_t *thr); nxt_int_t nxt_http_chunk_parse_test(nxt_thread_t *thr); nxt_int_t nxt_conf_json_depth_test(nxt_thread_t *thr); nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); +nxt_int_t nxt_conf_map_object_test(nxt_thread_t *thr); nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); nxt_int_t nxt_fd_event_change_test(nxt_thread_t *thr); nxt_int_t nxt_port_use_unless_zero_test(nxt_thread_t *thr); From f6171a2c1842a8bca7c7035117bb5333815b5ee1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 01:10:19 +0000 Subject: [PATCH 264/391] ci(sanitize): drop the fixed UBSan waivers, run the C tests under UBSan The nxt_conf.c (#188) and nxt_http_route.c (#200) reports are fixed by the previous commits. The nxt_string.h and nxt_tstr.h ones (#200) were fixed earlier, and their entries were kept until a clean sanitizer run. test_settings, test_configuration, test_python_basic, test_static and test_routing now run under the clang UBSan build with 0 reports. The file stays, with no entries: the sanitize scripts read it as optional. Build the c-tests-asan job with UBSan too, with -fno-sanitize-recover=undefined. nxt_conf_map_object_test then fails the job on a misaligned typed store instead of only logging it. freeunitorg/freeunit#188, freeunitorg/freeunit#200. Co-Authored-By: Claude Opus 5.5 --- .github/sanitizer-allowlist.txt | 26 ++------------------------ .github/workflows/build-test.yml | 8 +++++--- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/.github/sanitizer-allowlist.txt b/.github/sanitizer-allowlist.txt index 9e37d8ad1..fb4c3d71c 100644 --- a/.github/sanitizer-allowlist.txt +++ b/.github/sanitizer-allowlist.txt @@ -13,27 +13,5 @@ # - Pin the source location. A bare message would waive the same class of # violation everywhere in the tree, including a new one. # -# Measured on master (02263309) with the php leg's own test set: entries 1 and -# 2 fire on every run of test/test_php_application.py, entry 3 on the -# configuration-churn workload. - -# freeunitorg/freeunit#188 -- nxt_conf_value_t union read through an address -# the allocator only aligned to 4 bytes. Benign on x86-64, a real fault on -# strict-alignment targets; tracked for a fix in the value representation. -src/nxt_conf\.c:[0-9]+:[0-9]+: runtime error: member access within misaligned address 0x[0-9a-f]+ for type 'union ', which requires 8 byte alignment - -# freeunitorg/freeunit#200 -- nxt_strstr_eq() -> memcmp() on a zero-length -# nxt_str_t, whose .start is NULL. memcmp() declares both pointers nonnull -# regardless of length, so a zero-length compare is UB even though every -# implementation returns 0 without a dereference. -src/nxt_http_route\.c:[0-9]+:[0-9]+: runtime error: null pointer passed as argument [12], which is declared to never be null - -# freeunitorg/freeunit#200 -- same class in nxt_cpymem(): memcpy(dst, NULL, 0). -src/nxt_string\.h:[0-9]+:[0-9]+: runtime error: null pointer passed as argument 2, which is declared to never be null - -# freeunitorg/freeunit#200 -- same class in nxt_is_tstr(): memchr(NULL, c, 0) -# on a zero-length nxt_str_t, twice, one per character it looks for. Reached -# from the configuration validator. Measured with the python leg's test set: -# only test_configuration.py sends a configuration that gets there, so the -# entry arrived with that file. -src/nxt_tstr\.h:[0-9]+:[0-9]+: runtime error: null pointer passed as argument 1, which is declared to never be null +# No active waivers. The last four (freeunitorg/freeunit#188, #200) are +# fixed at their source. diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d4e715f04..b24582f2e 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -876,14 +876,16 @@ jobs: sudo apt-get -y update sudo apt-get -y install build-essential libssl-dev libpcre2-dev clang - - name: Configure unit (ASan+LSan, --tests) + # -fno-sanitize-recover: the first UBSan report fails the job. + # nxt_conf_map_object_test fails only under UBSan. + - name: Configure unit (ASan+LSan+UBSan, --tests) run: | ./configure \ --cc=clang \ --debug \ --tests \ - --cc-opt="-fsanitize=address -fno-omit-frame-pointer -g" \ - --ld-opt="-fsanitize=address" + --cc-opt="-fsanitize=address,undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer -g" \ + --ld-opt="-fsanitize=address,undefined" - name: Build C tests run: make -j4 tests From a7e12308fd82d9ad20c2a76d9cdecb658a593cb8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 02:51:29 +0000 Subject: [PATCH 265/391] test(conf): check what NXT_CONF_MAP_STR and NXT_CONF_MAP_PTR store The two non-scalar copies were only checked for their content. Read them back with memcpy() from their misaligned fields, and check that NXT_CONF_MAP_STR stores the member's own string, without a copy, that NXT_CONF_MAP_STR_COPY stores a copy, and that NXT_CONF_MAP_PTR stores the member's nxt_conf_value_t itself. freeunitorg/freeunit#188. Co-Authored-By: Claude Opus 5.5 --- src/test/nxt_conf_parse_test.c | 46 +++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/test/nxt_conf_parse_test.c b/src/test/nxt_conf_parse_test.c index e673c75a7..7e51e05da 100644 --- a/src/test/nxt_conf_parse_test.c +++ b/src/test/nxt_conf_parse_test.c @@ -177,11 +177,17 @@ typedef struct { nxt_int_t nxt_conf_map_object_test(nxt_thread_t *thr) { + void *ptr; nxt_mp_t *mp; + nxt_str_t str, rstr, orig; nxt_int_t ret; - nxt_conf_value_t *cv; + nxt_conf_value_t *cv, *member; nxt_conf_map_test_t dst; + static const nxt_str_t str_name = nxt_string("str"); + static const nxt_str_t rstr_name = nxt_string("rstr"); + static const nxt_str_t ptr_name = nxt_string("ptr"); + static const nxt_str_t json = nxt_string( "{\"flag\": true, \"i32\": -123456, \"i64\": -9876543210, \"i\": -7," " \"size\": 424242, \"off\": 131072, \"msec\": 3, \"dbl\": 1.5," @@ -250,6 +256,44 @@ nxt_conf_map_object_test(nxt_thread_t *thr) goto done; } + /* + * The non-scalar copies, read back with memcpy() from their misaligned + * fields. NXT_CONF_MAP_STR stores the member's own string, without a + * copy. NXT_CONF_MAP_STR_COPY stores a copy from the pool. + * NXT_CONF_MAP_PTR stores the member's nxt_conf_value_t itself. + */ + + nxt_memcpy(&rstr, &dst.rstr, sizeof(rstr)); + nxt_memcpy(&str, &dst.str, sizeof(str)); + nxt_memcpy(&ptr, &dst.ptr, sizeof(ptr)); + + member = nxt_conf_get_object_member(cv, &rstr_name, NULL); + nxt_conf_get_string(member, &orig); + + if (rstr.start != orig.start || rstr.length != orig.length) { + nxt_log_alert(thr->log, "nxt_conf_map_object() NXT_CONF_MAP_STR " + "did not store the member's own string"); + goto done; + } + + member = nxt_conf_get_object_member(cv, &str_name, NULL); + nxt_conf_get_string(member, &orig); + + if (str.start == orig.start + || str.length != orig.length + || memcmp(str.start, orig.start, orig.length) != 0) + { + nxt_log_alert(thr->log, "nxt_conf_map_object() NXT_CONF_MAP_STR_COPY " + "did not store a copy"); + goto done; + } + + if (ptr != nxt_conf_get_object_member(cv, &ptr_name, NULL)) { + nxt_log_alert(thr->log, "nxt_conf_map_object() NXT_CONF_MAP_PTR " + "did not store the member's value"); + goto done; + } + ret = NXT_OK; nxt_log_error(NXT_LOG_NOTICE, thr->log, From b0027c8efc4d5ec99adc3b6a9a38243902f1b60c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 02:51:29 +0000 Subject: [PATCH 266/391] string: short-circuit a zero length in nxt_str_eq() and nxt_str_start() memcmp() declares both pointers non-null whatever the length, and a zero-length nxt_str_t carries a NULL start. Short-circuit a zero length, as nxt_strstr_eq() already does. A zero length still compares equal. This is hardening. No call site passes a zero length with a NULL pointer today: nxt_str_eq() reaches memcmp() only on equal lengths, and every caller compares against a non-empty string. nxt_str_start() in nxt_php_sapi.c gets a zero length for a "root" of "/", but both pointers are non-null there, and the result is unchanged. freeunitorg/freeunit#200. Co-Authored-By: Claude Opus 5.5 --- src/nxt_string.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nxt_string.h b/src/nxt_string.h index 06201096a..2b3741003 100644 --- a/src/nxt_string.h +++ b/src/nxt_string.h @@ -148,12 +148,16 @@ NXT_EXPORT char *nxt_str_cstrz(nxt_mp_t *mp, const nxt_str_t *src); && (nxt_memcasecmp((s1)->start, (s2)->start, (s1)->length) == 0)) +/* The same zero-length short circuit as nxt_strstr_eq(), for memcmp(). */ + #define nxt_str_eq(s, p, _length) \ - (((s)->length == _length) && (memcmp((s)->start, p, _length) == 0)) + (((s)->length == _length) \ + && ((s)->length == 0 || memcmp((s)->start, p, _length) == 0)) #define nxt_str_start(s, p, _length) \ - (((s)->length >= _length) && (memcmp((s)->start, p, _length) == 0)) + (((s)->length >= _length) \ + && ((_length) == 0 || memcmp((s)->start, p, _length) == 0)) #define nxt_strchr_eq(s, c) \ From ec21b9beb4e40128f6cd64f0c1f5895fb860183e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 20:41:29 +0000 Subject: [PATCH 267/391] conn: make nxt_conn_close() safe for queued I/O handlers An I/O handler queued in the same engine cycle as nxt_conn_close() cannot be taken back, and the guards against it were left to the callers, unevenly. Findings M-1, M-2, M-3 and M-9 of the conn FSM audit: M-1: nxt_conn_io_write() tested socket.error and block_write before c->write == NULL, so a write queued before the close took the error path and queued the close state's NULL error_handler, which the engine then called. nxt_h1p_peer_close() and the h1p timeouts reach this. Test c->write == NULL first. M-2: nxt_conn_close() did not block reads, so a queued read read the closing fd and ran the read state handlers, which in h1proto close the connection again. nxt_conn_close() now sets block_read, and the kqueue read wrapper honours it before its EOF fast path. M-3: a second nxt_conn_close() queued a second close, and the release handler freed the conn twice. A new "closing" flag, in an existing padding hole of nxt_conn_t, makes the second call a no-op. M-9: nxt_conn_free() asserts that the socket is already closed. The comment in nxt_h1p_peer_close() wrongly said that block_write makes a queued write bail out early; it now says what stops each handler. nxt_conn_close_test covers the write, read and double close cases on an epoll engine over a socketpair; all three fail without this change. Reference: https://github.com/freeunitorg/freeunit/pull/464 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FXFwqBMUbo35vhzidxWQpp --- auto/sources | 1 + src/nxt_conn.c | 3 + src/nxt_conn.h | 3 + src/nxt_conn_close.c | 19 +- src/nxt_conn_write.c | 10 +- src/nxt_h1proto.c | 8 +- src/nxt_kqueue_engine.c | 5 + src/test/nxt_conn_close_test.c | 471 +++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 10 files changed, 517 insertions(+), 8 deletions(-) create mode 100644 src/test/nxt_conn_close_test.c diff --git a/auto/sources b/auto/sources index 9d727eba1..6e0b9fcb9 100644 --- a/auto/sources +++ b/auto/sources @@ -191,6 +191,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_fd_test.c \ src/test/nxt_port_rpc_fd_test.c \ src/test/nxt_port_queued_fd_test.c \ + src/test/nxt_conn_close_test.c \ " diff --git a/src/nxt_conn.c b/src/nxt_conn.c index ef10c2103..adc1ad6d8 100644 --- a/src/nxt_conn.c +++ b/src/nxt_conn.c @@ -160,6 +160,9 @@ nxt_conn_free(nxt_task_t *task, nxt_conn_t *c) nxt_assert(c->task.thread == task->thread); + /* An open fd would leak and leave a stale fd-event registered. */ + nxt_assert(c->socket.fd == -1); + /* * Take both timers out of the engine's timer machinery, and take the conn * off whichever tracking queue it is on. Callers only ever diff --git a/src/nxt_conn.h b/src/nxt_conn.h index e146a05b2..4bb51c863 100644 --- a/src/nxt_conn.h +++ b/src/nxt_conn.h @@ -167,6 +167,9 @@ struct nxt_conn_s { uint8_t block_write; /* 1 bit */ uint8_t delayed; /* 1 bit */ + /* Makes a repeated nxt_conn_close() a no-op; never cleared. */ + uint8_t closing; /* 1 bit */ + #define NXT_CONN_TRACK_NONE 0 #define NXT_CONN_TRACK_IDLE 1 #define NXT_CONN_TRACK_ACTIVE 2 diff --git a/src/nxt_conn_close.c b/src/nxt_conn_close.c index 80e28fc34..f5fed1412 100644 --- a/src/nxt_conn_close.c +++ b/src/nxt_conn_close.c @@ -27,13 +27,24 @@ nxt_conn_close(nxt_event_engine_t *engine, nxt_conn_t *c) .l_linger = 0, }; - nxt_debug(c->socket.task, "conn close fd:%d, to:%d", - c->socket.fd, c->socket.timedout); + nxt_debug(c->socket.task, "conn close fd:%d, to:%d, cl:%d", + c->socket.fd, c->socket.timedout, c->closing); /* - * Disable all pending write operations because on success they - * will incorrectly call a ready handler set for nxt_conn_close(). + * A handler queued before the first close may close again; a second + * close handler would run the release handler twice. */ + if (c->closing) { + return; + } + + c->closing = 1; + + /* + * Read or write handlers already queued must not touch the socket or + * call the previous state's handlers. + */ + c->block_read = 1; c->write = NULL; if (c->socket.timedout) { diff --git a/src/nxt_conn_write.c b/src/nxt_conn_write.c index c29a83f89..a955fd676 100644 --- a/src/nxt_conn_write.c +++ b/src/nxt_conn_write.c @@ -27,11 +27,19 @@ nxt_conn_io_write(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "conn write fd:%d er:%d bl:%d", c->socket.fd, c->socket.error, c->block_write); + /* + * Test this first: after nxt_conn_close() the error path would queue + * the close state's NULL error_handler. + */ + if (c->write == NULL) { + return; + } + if (c->socket.error != 0 || c->block_write) { goto error; } - if (!c->socket.write_ready || c->write == NULL) { + if (!c->socket.write_ready) { return; } diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 66e82d75f..b2497de2d 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -3381,9 +3381,11 @@ nxt_h1p_peer_close(nxt_task_t *task, nxt_http_peer_t *peer) * nxt_h1p_peer_read_done()/nxt_h1p_peer_send_timeout()/etc. and dereference * the freed peer -- a use-after-free that crashes the router. Both paths * are at risk: the read side (response relay) and the write side (the - * request body upload uses an autoreset send timer). Setting block_read / - * block_write makes a queued nxt_conn_io_read()/nxt_conn_io_write() bail out - * early; nxt_conn_close() still emits the FIN via its work-queue handler. + * request body upload uses an autoreset send timer). block_read stops + * a queued nxt_conn_io_read(), and a queued nxt_conn_io_write() returns + * because nxt_conn_close() clears c->write. The flags are set here for + * the fd == -1 branch, which skips nxt_conn_close(). nxt_conn_close() + * still emits the FIN via its work-queue handler. */ c->block_read = 1; c->block_write = 1; diff --git a/src/nxt_kqueue_engine.c b/src/nxt_kqueue_engine.c index d3b8c3be6..033a91943 100644 --- a/src/nxt_kqueue_engine.c +++ b/src/nxt_kqueue_engine.c @@ -1060,6 +1060,11 @@ nxt_kqueue_conn_io_read(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "kqueue conn read fd:%d", c->socket.fd); + /* As in nxt_conn_io_read(): a closed conn must not reach the EOF path. */ + if (c->socket.error != 0 || c->block_read) { + return; + } + if (c->socket.kq_available == 0 && c->socket.kq_eof) { nxt_debug(task, "kevent fd:%d eof", c->socket.fd); diff --git a/src/test/nxt_conn_close_test.c b/src/test/nxt_conn_close_test.c new file mode 100644 index 000000000..614535607 --- /dev/null +++ b/src/test/nxt_conn_close_test.c @@ -0,0 +1,471 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * An I/O handler queued before nxt_conn_close() in the same engine cycle + * cannot be taken back, so it must be harmless. Each leg queues handlers + * on a real conn over a socketpair, closes, and drains the work queues in + * engine order. The fd is never registered, so the close is synchronous. + */ + +#include +#include "nxt_tests.h" + +#include +#include + + +#if (NXT_HAVE_EPOLL_EDGE) + +typedef struct { + nxt_event_engine_t *engine; + nxt_event_engine_t *saved_engine; + nxt_conn_t *c; + nxt_fd_t peer; + + nxt_uint_t released; + nxt_uint_t read_handlers; + nxt_uint_t null_handlers; + nxt_fd_t fd_at_release; +} nxt_conn_close_test_t; + + +static nxt_int_t nxt_conn_close_test_write(nxt_thread_t *thr); +static nxt_int_t nxt_conn_close_test_read(nxt_thread_t *thr); +static nxt_int_t nxt_conn_close_test_twice(nxt_thread_t *thr); +static nxt_int_t nxt_conn_close_test_setup(nxt_thread_t *thr, + nxt_conn_close_test_t *t); +static nxt_int_t nxt_conn_close_test_finish(nxt_thread_t *thr, + nxt_conn_close_test_t *t, nxt_int_t ret); +static void nxt_conn_close_test_drain(nxt_conn_close_test_t *t); +static nxt_bool_t nxt_conn_close_test_drain_wq(nxt_conn_close_test_t *t, + nxt_work_queue_t *wq); +static void nxt_conn_close_test_release(nxt_task_t *task, void *obj, + void *data); +static void nxt_conn_close_test_read_handler(nxt_task_t *task, void *obj, + void *data); + + +static nxt_conn_close_test_t nxt_conn_close_test_fixture; + + +/* Like every close state in the tree: no error_handler. */ + +static const nxt_conn_state_t nxt_conn_close_test_close_state + nxt_aligned(64) = +{ + .ready_handler = nxt_conn_close_test_release, +}; + + +static const nxt_conn_state_t nxt_conn_close_test_read_state + nxt_aligned(64) = +{ + .ready_handler = nxt_conn_close_test_read_handler, + .close_handler = nxt_conn_close_test_read_handler, + .error_handler = nxt_conn_close_test_read_handler, +}; + +#endif + + +nxt_int_t +nxt_conn_close_test(nxt_thread_t *thr) +{ +#if (NXT_HAVE_EPOLL_EDGE) + + nxt_int_t ret; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn close test started"); + + ret = nxt_conn_close_test_write(thr); + + if (nxt_conn_close_test_read(thr) != NXT_OK) { + ret = NXT_ERROR; + } + + if (nxt_conn_close_test_twice(thr) != NXT_OK) { + ret = NXT_ERROR; + } + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn close test passed"); + +#endif + + return NXT_OK; +} + + +#if (NXT_HAVE_EPOLL_EDGE) + +/* Queues as nxt_h1p_conn_init() and the fd dispatch set them. */ + +static nxt_int_t +nxt_conn_close_test_setup(nxt_thread_t *thr, nxt_conn_close_test_t *t) +{ + nxt_mp_t *mp; + nxt_fd_t pair[2]; + nxt_conn_t *c; + nxt_task_t *task; + nxt_event_engine_t *engine; + + task = thr->task; + task->thread = thr; + + nxt_memzero(t, sizeof(nxt_conn_close_test_t)); + t->peer = -1; + t->fd_at_release = -2; + + t->saved_engine = thr->engine; + + engine = nxt_event_engine_create(task, &nxt_epoll_edge_engine, NULL, 0, 0); + if (nxt_slow_path(engine == NULL)) { + return NXT_ERROR; + } + + t->engine = engine; + thr->engine = engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + c = nxt_conn_create(mp, task); + if (nxt_slow_path(c == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + t->c = c; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_STREAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: socketpair failed"); + return NXT_ERROR; + } + + if (nxt_slow_path(fcntl(pair[0], F_SETFL, O_NONBLOCK) == -1 + || fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) + { + nxt_fd_close(pair[0]); + nxt_fd_close(pair[1]); + return NXT_ERROR; + } + + c->socket.fd = pair[0]; + t->peer = pair[1]; + + c->socket.read_ready = 1; + c->socket.write_ready = 1; + + c->read_work_queue = &engine->fast_work_queue; + c->write_work_queue = &engine->fast_work_queue; + c->socket.read_work_queue = &engine->read_work_queue; + c->socket.write_work_queue = &engine->write_work_queue; + + c->read = nxt_buf_mem_alloc(mp, 64, 0); + if (nxt_slow_path(c->read == NULL)) { + return NXT_ERROR; + } + + c->read_state = &nxt_conn_close_test_read_state; + c->write_state = &nxt_conn_close_test_close_state; + + return NXT_OK; +} + + +static nxt_int_t +nxt_conn_close_test_finish(nxt_thread_t *thr, nxt_conn_close_test_t *t, + nxt_int_t ret) +{ + nxt_conn_t *c; + + c = t->c; + + if (c != NULL) { + if (c->socket.fd != -1) { + nxt_fd_close(c->socket.fd); + c->socket.fd = -1; + } + + nxt_conn_free(thr->task, c); + } + + if (t->peer != -1) { + nxt_fd_close(t->peer); + } + + if (t->engine != NULL) { + nxt_event_engine_free(t->engine); + } + + thr->engine = t->saved_engine; + + return ret; +} + + +/* + * Drain in nxt_event_engine_start() order, firing close timers between + * passes. A NULL handler is counted instead of called. + */ + +static void +nxt_conn_close_test_drain(nxt_conn_close_test_t *t) +{ + nxt_bool_t ran; + nxt_event_engine_t *engine; + + engine = t->engine; + + for ( ;; ) { + do { + ran = nxt_conn_close_test_drain_wq(t, &engine->fast_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->accept_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->read_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->socket_work_queue) + || nxt_conn_close_test_drain_wq(t, + &engine->connect_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->write_work_queue) + || nxt_conn_close_test_drain_wq(t, + &engine->shutdown_work_queue) + || nxt_conn_close_test_drain_wq(t, &engine->close_work_queue); + } while (ran); + + nxt_timer_find(engine); + + if (nxt_rbtree_is_empty(&engine->timers.tree)) { + return; + } + + nxt_timer_expire(engine, engine->timers.now + 100); + + if (engine->fast_work_queue.head == NULL) { + return; + } + } +} + + +static nxt_bool_t +nxt_conn_close_test_drain_wq(nxt_conn_close_test_t *t, nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *task; + nxt_work_handler_t handler; + + if (wq->head == NULL) { + return 0; + } + + handler = nxt_work_queue_pop(wq, &task, &obj, &data); + + if (handler == NULL) { + t->null_handlers++; + return 1; + } + + handler(task, obj, data); + + return 1; +} + + +static void +nxt_conn_close_test_release(nxt_task_t *task, void *obj, void *data) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + c = obj; + t = &nxt_conn_close_test_fixture; + + t->released++; + t->fd_at_release = c->socket.fd; +} + + +/* Like the h1proto read handlers, which end in nxt_h1p_shutdown(). */ + +static void +nxt_conn_close_test_read_handler(nxt_task_t *task, void *obj, void *data) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + c = obj; + t = &nxt_conn_close_test_fixture; + + t->read_handlers++; + + nxt_conn_close(task->thread->engine, c); +} + + +/* + * A queued write, then block_write and close, as nxt_h1p_peer_close() and + * the h1p timeouts do: the close state's NULL error_handler must not run. + */ + +static nxt_int_t +nxt_conn_close_test_write(nxt_thread_t *thr) +{ + ssize_t n; + u_char byte; + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + t = &nxt_conn_close_test_fixture; + + if (nxt_conn_close_test_setup(thr, t) != NXT_OK) { + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + c = t->c; + + c->write = c->read; + + nxt_work_queue_add(c->socket.write_work_queue, nxt_conn_io_write, + c->socket.task, c, c->socket.data); + + c->block_write = 1; + + nxt_conn_close(t->engine, c); + + nxt_conn_close_test_drain(t); + + if (t->null_handlers != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: write: a queued nxt_conn_io_write() " + "on a closed conn queued the NULL error_handler of the " + "close state (%ui times); the engine would call it", + t->null_handlers); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + if (t->released != 1 || t->fd_at_release != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: write: release handler ran %ui " + "times (expected 1), fd at release %d (expected -1)", + t->released, (int) t->fd_at_release); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + /* Nothing was written; the peer sees only the FIN. */ + + n = recv(t->peer, &byte, 1, 0); + + if (n != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: write: peer recv() %z, expected 0", + n); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + return nxt_conn_close_test_finish(thr, t, NXT_OK); +} + + +/* + * A queued read, then a close without block_read, as nxt_h1p_conn_close() + * does: nothing may be read and no read state handler may run. + */ + +static nxt_int_t +nxt_conn_close_test_read(nxt_thread_t *thr) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + t = &nxt_conn_close_test_fixture; + + if (nxt_conn_close_test_setup(thr, t) != NXT_OK) { + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + c = t->c; + + if (send(t->peer, "ping", 4, 0) != 4) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: read: send() failed"); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + nxt_work_queue_add(c->socket.read_work_queue, nxt_conn_io_read, + c->socket.task, c, c->socket.data); + + nxt_conn_close(t->engine, c); + + nxt_conn_close_test_drain(t); + + if (t->read_handlers != 0 || c->nbytes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: read: a queued nxt_conn_io_read() " + "on a closed conn read %uD bytes and ran %ui read " + "state handlers (expected 0 and 0)", + c->nbytes, t->read_handlers); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + if (t->null_handlers != 0 || t->released != 1 || t->fd_at_release != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: read: %ui NULL handlers, release " + "handler ran %ui times (expected 1), fd at release %d " + "(expected -1)", t->null_handlers, t->released, + (int) t->fd_at_release); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + return nxt_conn_close_test_finish(thr, t, NXT_OK); +} + + +/* Two closes in one cycle must release once. */ + +static nxt_int_t +nxt_conn_close_test_twice(nxt_thread_t *thr) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + t = &nxt_conn_close_test_fixture; + + if (nxt_conn_close_test_setup(thr, t) != NXT_OK) { + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + c = t->c; + + nxt_conn_close(t->engine, c); + nxt_conn_close(t->engine, c); + + nxt_conn_close_test_drain(t); + + if (t->released != 1 || t->fd_at_release != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: twice: release handler ran %ui " + "times (expected 1), fd at release %d (expected -1)", + t->released, (int) t->fd_at_release); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + if (c->socket.fd != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: twice: fd %d still open after the " + "close", (int) c->socket.fd); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + return nxt_conn_close_test_finish(thr, t, NXT_OK); +} + +#endif diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 5eab4b6e1..41cd2150d 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -290,6 +290,10 @@ main(int argc, char **argv) return 1; } + if (nxt_conn_close_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CGROUP) if (nxt_cgroup_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index adb6bb37b..c6cf4c5a3 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -94,6 +94,7 @@ nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_port_rpc_fd_test(nxt_thread_t *thr); nxt_int_t nxt_port_queued_fd_test(nxt_thread_t *thr); +nxt_int_t nxt_conn_close_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From 498cb076d29f0e567be91f992bf88d1d15c6f248 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 21:15:58 +0000 Subject: [PATCH 268/391] fix(conn): return early from a queued write only on a closed conn nxt_conn_io_write() tested c->write == NULL before socket.error and block_write to stop a write queued before nxt_conn_close() from queuing the close state's NULL error_handler (M-1). That also changed conns that are not closing: with socket.error set and no c->write the write handler returned without calling error_handler. Test the closing flag first instead, and keep the old order of the other checks. nxt_h1p_peer_close() set block_write for its fd == -1 branch, which skips nxt_conn_close(). block_write does not stop a queued write: it sends it to the write state's error_handler, which uses the peer that is freed right after. That branch now sets closing, so a queued write returns, and the comment says what stops each handler. Behaviour change on kqueue (BSD, macOS), from the parent commit: nxt_kqueue_conn_io_read() now returns when socket.error is set. Before, a kevent EOF with an error queued read_state->close_handler from the wrapper, and the error path ran as well. Now only the error path runs. Reference: https://github.com/freeunitorg/freeunit/pull/464 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- src/nxt_conn.c | 2 +- src/nxt_conn_write.c | 8 ++++---- src/nxt_h1proto.c | 12 +++++++----- src/nxt_kqueue_engine.c | 6 +++++- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/nxt_conn.c b/src/nxt_conn.c index adc1ad6d8..8c796407b 100644 --- a/src/nxt_conn.c +++ b/src/nxt_conn.c @@ -192,7 +192,7 @@ nxt_conn_free(nxt_task_t *task, nxt_conn_t *c) /* * Park the struct rather than publishing it to the freelist directly. * A conn can be freed while work items that reference it are still - * queued -- nxt_h1p_peer_close() sets block_read/block_write for + * queued -- nxt_h1p_peer_close() sets block_read/closing for * exactly that reason, then reaches nxt_conn_free() synchronously on * its fd == -1 path. Handing the struct out now would let * nxt_conn_create()'s re-zero clear those guards from under the queued diff --git a/src/nxt_conn_write.c b/src/nxt_conn_write.c index a955fd676..6ccff40ff 100644 --- a/src/nxt_conn_write.c +++ b/src/nxt_conn_write.c @@ -28,10 +28,10 @@ nxt_conn_io_write(nxt_task_t *task, void *obj, void *data) c->socket.fd, c->socket.error, c->block_write); /* - * Test this first: after nxt_conn_close() the error path would queue - * the close state's NULL error_handler. + * A write queued before nxt_conn_close() returns here: the error path + * would queue the close state's NULL error_handler. */ - if (c->write == NULL) { + if (c->closing) { return; } @@ -39,7 +39,7 @@ nxt_conn_io_write(nxt_task_t *task, void *obj, void *data) goto error; } - if (!c->socket.write_ready) { + if (!c->socket.write_ready || c->write == NULL) { return; } diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index b2497de2d..d79d69c91 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -3382,13 +3382,13 @@ nxt_h1p_peer_close(nxt_task_t *task, nxt_http_peer_t *peer) * the freed peer -- a use-after-free that crashes the router. Both paths * are at risk: the read side (response relay) and the write side (the * request body upload uses an autoreset send timer). block_read stops - * a queued nxt_conn_io_read(), and a queued nxt_conn_io_write() returns - * because nxt_conn_close() clears c->write. The flags are set here for - * the fd == -1 branch, which skips nxt_conn_close(). nxt_conn_close() - * still emits the FIN via its work-queue handler. + * a queued nxt_conn_io_read(), and the closing flag makes a queued + * nxt_conn_io_write() return. nxt_conn_close() sets both; the fd == -1 + * branch skips it and sets them here. block_write would not do: it sends + * a queued write to the write state's error_handler, which uses the peer. + * nxt_conn_close() still emits the FIN via its work-queue handler. */ c->block_read = 1; - c->block_write = 1; nxt_timer_disable(task->thread->engine, &c->read_timer); nxt_timer_disable(task->thread->engine, &c->write_timer); @@ -3398,6 +3398,8 @@ nxt_h1p_peer_close(nxt_task_t *task, nxt_http_peer_t *peer) nxt_conn_close(task->thread->engine, c); } else { + c->closing = 1; + nxt_h1p_peer_free(task, c, NULL); } } diff --git a/src/nxt_kqueue_engine.c b/src/nxt_kqueue_engine.c index 033a91943..a39fbdaff 100644 --- a/src/nxt_kqueue_engine.c +++ b/src/nxt_kqueue_engine.c @@ -1060,7 +1060,11 @@ nxt_kqueue_conn_io_read(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "kqueue conn read fd:%d", c->socket.fd); - /* As in nxt_conn_io_read(): a closed conn must not reach the EOF path. */ + /* + * As in nxt_conn_io_read(): a closed conn must not reach the EOF path. + * An EOF with socket.error set runs only the error path, not also the + * read state's close_handler. + */ if (c->socket.error != 0 || c->block_read) { return; } From c0a18d13c26cd362e2ea7e06b3db388c2f5e84d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 21:15:58 +0000 Subject: [PATCH 269/391] test(conn): cover a close from a queued read after a close A read ran and queued its ready handler, a new read event is queued, then the conn is closed. The ready handler closes again and the new read would reach the FIN and close once more. The leg fails without the closing guard in nxt_conn_close() or without block_read. Also note that the drain loop relies on the 0 ms close timer. Reference: https://github.com/freeunitorg/freeunit/pull/464 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- src/test/nxt_conn_close_test.c | 82 ++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/src/test/nxt_conn_close_test.c b/src/test/nxt_conn_close_test.c index 614535607..203f024ca 100644 --- a/src/test/nxt_conn_close_test.c +++ b/src/test/nxt_conn_close_test.c @@ -34,6 +34,7 @@ typedef struct { static nxt_int_t nxt_conn_close_test_write(nxt_thread_t *thr); static nxt_int_t nxt_conn_close_test_read(nxt_thread_t *thr); static nxt_int_t nxt_conn_close_test_twice(nxt_thread_t *thr); +static nxt_int_t nxt_conn_close_test_race(nxt_thread_t *thr); static nxt_int_t nxt_conn_close_test_setup(nxt_thread_t *thr, nxt_conn_close_test_t *t); static nxt_int_t nxt_conn_close_test_finish(nxt_thread_t *thr, @@ -90,6 +91,10 @@ nxt_conn_close_test(nxt_thread_t *thr) ret = NXT_ERROR; } + if (nxt_conn_close_test_race(thr) != NXT_OK) { + ret = NXT_ERROR; + } + if (ret != NXT_OK) { return NXT_ERROR; } @@ -247,6 +252,10 @@ nxt_conn_close_test_drain(nxt_conn_close_test_t *t) return; } + /* + * nxt_conn_close_handler() adds the close timer with 0 ms, so a + * 100 ms step fires it without advancing the clock. + */ nxt_timer_expire(engine, engine->timers.now + 100); if (engine->fast_work_queue.head == NULL) { @@ -468,4 +477,77 @@ nxt_conn_close_test_twice(nxt_thread_t *thr) return nxt_conn_close_test_finish(thr, t, NXT_OK); } + +/* + * The full race: a read ran and queued its ready handler, a new read event + * is queued, then the conn is closed. The ready handler closes again (the + * second close must be a no-op) and the new read must not reach the FIN + * and run the close handler, which would close once more. + */ + +static nxt_int_t +nxt_conn_close_test_race(nxt_thread_t *thr) +{ + nxt_conn_t *c; + nxt_conn_close_test_t *t; + + t = &nxt_conn_close_test_fixture; + + if (nxt_conn_close_test_setup(thr, t) != NXT_OK) { + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + c = t->c; + + if (send(t->peer, "ping", 4, 0) != 4 || shutdown(t->peer, SHUT_WR) != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: race: send() or shutdown() failed"); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + /* The first read runs and queues the ready handler. */ + + nxt_work_queue_add(c->socket.read_work_queue, nxt_conn_io_read, + c->socket.task, c, c->socket.data); + + (void) nxt_conn_close_test_drain_wq(t, c->socket.read_work_queue); + + if (c->nbytes != 4 || c->read_work_queue->head == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: race: the first read got %uD bytes " + "(expected 4) or queued no handler", c->nbytes); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + /* A new edge event: the FIN is readable. */ + + c->socket.read_ready = 1; + + nxt_work_queue_add(c->socket.read_work_queue, nxt_conn_io_read, + c->socket.task, c, c->socket.data); + + nxt_conn_close(t->engine, c); + + nxt_conn_close_test_drain(t); + + if (t->read_handlers != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: race: %ui read state handlers ran " + "(expected 1); the queued read after the close reached " + "the socket", t->read_handlers); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + if (t->null_handlers != 0 || t->released != 1 || t->fd_at_release != -1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close test: race: %ui NULL handlers, release " + "handler ran %ui times (expected 1), fd at release %d " + "(expected -1)", t->null_handlers, t->released, + (int) t->fd_at_release); + return nxt_conn_close_test_finish(thr, t, NXT_ERROR); + } + + return nxt_conn_close_test_finish(thr, t, NXT_OK); +} + #endif From 60b789e607c770a688b4f7a7bc6c7187463f88ad Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 04:03:58 +0000 Subject: [PATCH 270/391] proxy: consume upstream 1xx interim responses nxt_h1p_peer_header_parse() took the first status line from the upstream as the response. A 1xx is an interim response (RFC 9110, 15.2), and the final response comes after it on the same connection. So "100 Continue" or "103 Early Hints" was sent to the client as the response, and the final response was sent as its body, or as the answer to a pipelined request. Unit can cause the 100 itself: it forwards a client's "Expect: 100-continue" to the upstream as is. Now the parser drops each 1xx except 101 and reads the next status line. The client gets only the final response. 101 changes the protocol, so it is still the response. - The fields of a 1xx are checked but not stored ("discard_fields"). - The bytes of a 1xx are freed from the header buffer, so the final header can use the whole buffer. - More than 10 interim responses give 502, as in Apache. - A 1xx bigger than the header buffer gives 502. Before, it was sent to the client as the response. Sending 103 to the client is out of scope: the client side cannot yet send a header that does not complete the response. test/test_proxy_interim.py covers 100, 103, several 1xx in one or more reads, an empty line split between two reads, a big 1xx, "Expect", fields and framing fields of a 1xx, 101, a close after a 1xx, 10 and 11 1xx responses, and a 1xx bigger than the buffer. nxt_http_parse_test.c covers "discard_fields". Co-Authored-By: Claude Opus 5.5 --- CHANGES | 6 + docs/changes.xml | 10 + src/nxt_h1proto.c | 68 +++++-- src/nxt_http.h | 2 + src/nxt_http_parse.c | 2 +- src/nxt_http_parse.h | 2 + src/test/nxt_http_parse_test.c | 92 +++++++++ test/fake_upstream/README.md | 4 +- test/test_proxy_head.py | 4 +- test/test_proxy_interim.py | 351 +++++++++++++++++++++++++++++++++ 10 files changed, 525 insertions(+), 16 deletions(-) create mode 100644 test/test_proxy_interim.py diff --git a/CHANGES b/CHANGES index db122d6f9..6fb2dc96a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: an upstream 1xx interim response ("100 Continue", "103 Early + Hints") was relayed to the client as the response, with the final + response relayed as its body. The proxy now drops interim responses + (101 excepted) and relays the final one; more than ten of them, or + one larger than the proxy header buffer, fails the request with 502. + *) Change: a certificate bundle whose private key does not belong to its first certificate is now refused at upload with 400 "Invalid certificate.". Before, the bundle was stored, and the mismatch diff --git a/docs/changes.xml b/docs/changes.xml index 5ffebb77c..2b48f85bf 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,16 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +an upstream 1xx interim response ("100 Continue", "103 Early Hints") was +relayed to the client as the response, with the final response relayed as its +body. The proxy now drops interim responses (101 excepted) and relays the +final one; more than ten of them, or one larger than the proxy header buffer, +fails the request with 502. + + + a certificate bundle whose private key does not belong to its first diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index d79d69c91..aa783d01a 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -2920,13 +2920,8 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) * releases the request pool. * * The predicate is the "final response" one, not the full RFC list: - * a 1xx from an upstream is an interim response, and nothing here - * continues the exchange past it -- nxt_h1p_peer_header_parse() only - * reads a status line while peer->status is still NXT_HTTP_UNSET, so - * the 1xx is taken as the response and whatever follows is relayed as - * its body. That is pre-existing and out of scope; ending the - * exchange on the 1xx header here would discard a final response that - * may already be sitting in this very buffer. + * nxt_h1p_peer_header_parse() drops a 1xx and reads on, so a 1xx + * never reaches here. * * "b" is not forwarded: bytes an upstream put after the header of a * bodyless response are not a body. It is handed to @@ -3004,12 +2999,28 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) } +/* + * A 1xx other than 101 is an interim response (RFC 9110, 15.2). + * Limit how many one upstream response may have. Apache also uses 10. + */ +#define NXT_HTTP_MAX_INTERIM_RESPONSES 10 + +#define nxt_h1p_peer_status_interim(status) \ + ((status) >= NXT_HTTP_CONTINUE && (status) < NXT_HTTP_OK \ + && (status) != NXT_HTTP_SWITCHING_PROTOCOLS) + + static nxt_int_t nxt_h1p_peer_header_parse(nxt_http_peer_t *peer, nxt_buf_mem_t *bm) { - u_char *p; - size_t length; - nxt_int_t status; + u_char *p; + size_t length; + nxt_int_t ret, status; + nxt_http_request_parse_t *rp; + + rp = &peer->proto.h1->parser; + +again: if (peer->status < 0) { length = nxt_buf_mem_used_size(bm); @@ -3043,9 +3054,44 @@ nxt_h1p_peer_header_parse(nxt_http_peer_t *peer, nxt_buf_mem_t *bm) bm->pos = p + 1; peer->status = status; + + /* Do not store the fields of a 1xx. */ + rp->discard_fields = nxt_h1p_peer_status_interim(status); + } + + ret = nxt_http_parse_fields(rp, bm); + + if (ret != NXT_DONE || !nxt_h1p_peer_status_interim(peer->status)) { + return ret; } - return nxt_http_parse_fields(&peer->proto.h1->parser, bm); + /* + * Drop the 1xx and read the next response. The client gets only + * the final response. NXT_ERROR becomes 502. + */ + if (nxt_slow_path(++peer->num_interim > NXT_HTTP_MAX_INTERIM_RESPONSES)) { + nxt_log(&peer->request->task, NXT_LOG_WARN, + "upstream sent more than %d interim responses", + NXT_HTTP_MAX_INTERIM_RESPONSES); + + return NXT_ERROR; + } + + /* The handler can point to the end of the empty line. */ + rp->handler = NULL; + peer->status = NXT_HTTP_UNSET; + + /* + * Free the 1xx bytes, so the final header can use the whole buffer. + * This is safe only after NXT_DONE: no pointer into the buffer is left. + */ + length = bm->free - bm->pos; + nxt_memmove(bm->start, bm->pos, length); + + bm->pos = bm->start; + bm->free = bm->start + length; + + goto again; } diff --git a/src/nxt_http.h b/src/nxt_http.h index fddc3f0c0..b84f86d4c 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -176,6 +176,8 @@ typedef struct { nxt_http_protocol_t protocol:8; /* 2 bits */ uint8_t header_received; /* 1 bit */ uint8_t closed; /* 1 bit */ + /* number of 1xx responses dropped */ + uint8_t num_interim; } nxt_http_peer_t; diff --git a/src/nxt_http_parse.c b/src/nxt_http_parse.c index e8f71334c..224ca3f7c 100644 --- a/src/nxt_http_parse.c +++ b/src/nxt_http_parse.c @@ -785,7 +785,7 @@ nxt_http_parse_field_end(nxt_http_request_parse_t *rp, u_char **pos, if (rp->skip_field) { rp->skip_field = 0; - } else { + } else if (!rp->discard_fields) { if (rp->num_inline_fields < 16) { field = &rp->inline_fields[rp->num_inline_fields++]; diff --git a/src/nxt_http_parse.h b/src/nxt_http_parse.h index c0d5c6226..150929684 100644 --- a/src/nxt_http_parse.h +++ b/src/nxt_http_parse.h @@ -72,6 +72,8 @@ struct nxt_http_request_parse_s { uint8_t skip_field; /* 1 bit */ uint8_t discard_unsafe_fields; /* 1 bit */ + /* check the fields, but do not store them */ + uint8_t discard_fields; /* 1 bit */ /* target with "/." */ uint8_t complex_target; /* 1 bit */ diff --git a/src/test/nxt_http_parse_test.c b/src/test/nxt_http_parse_test.c index 6c81c0b5e..7c6bc4ca1 100644 --- a/src/test/nxt_http_parse_test.c +++ b/src/test/nxt_http_parse_test.c @@ -70,6 +70,7 @@ static nxt_int_t nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, nxt_str_t *request, nxt_log_t *log); static nxt_int_t nxt_http_parse_test_fields(nxt_http_request_parse_t *rp, nxt_http_parse_test_data_t *data, nxt_str_t *request, nxt_log_t *log); +static nxt_int_t nxt_http_parse_test_discard_fields(nxt_thread_t *thr); static nxt_int_t nxt_http_parse_test_headers(nxt_http_request_parse_t *rp, nxt_http_parse_test_data_t *data, nxt_str_t *request, nxt_log_t *log); @@ -802,6 +803,10 @@ nxt_http_parse_test(nxt_thread_t *thr) nxt_mp_destroy(mp_temp); } + if (nxt_http_parse_test_discard_fields(thr) != NXT_OK) { + return NXT_ERROR; + } + nxt_log_error(NXT_LOG_NOTICE, thr->log, "http parse test passed"); nxt_memzero(&hash, sizeof(nxt_lvlhsh_t)); @@ -856,6 +861,93 @@ nxt_http_parse_test(nxt_thread_t *thr) } +/* + * With "discard_fields", fields are checked but not stored. Then a reset of + * the handler is enough to parse the next block. 24 fields is more than the + * 16 inline fields. + */ + +static nxt_int_t +nxt_http_parse_test_discard_fields(nxt_thread_t *thr) +{ + u_char *p; + nxt_mp_t *mp; + nxt_int_t rc, ret; + nxt_uint_t i, nfields; + nxt_buf_mem_t buf; + nxt_http_field_t *field; + nxt_http_request_parse_t rp; + u_char block[24 * nxt_length("X-Field-00: v\r\n") + 2]; + + p = block; + + for (i = 0; i < 24; i++) { + p = nxt_sprintf(p, block + sizeof(block), "X-Field-%02ui: v\r\n", i); + } + + *p++ = '\r'; + *p++ = '\n'; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + + nxt_memzero(&rp, sizeof(nxt_http_request_parse_t)); + + if (nxt_http_parse_request_init(&rp, mp) != NXT_OK) { + goto done; + } + + rp.discard_fields = 1; + + buf.start = block; + buf.pos = block; + buf.free = p; + buf.end = p; + + rc = nxt_http_parse_fields(&rp, &buf); + + if (rc != NXT_DONE || rp.num_inline_fields != 0 || rp.fields != NULL) { + nxt_log_alert(thr->log, "http parse discard fields test failed: " + "rc %i, %ui inline fields, list %p", + rc, (nxt_uint_t) rp.num_inline_fields, rp.fields); + goto done; + } + + rp.handler = NULL; + rp.discard_fields = 0; + + buf.pos = block; + + rc = nxt_http_parse_fields(&rp, &buf); + + nfields = 0; + + nxt_http_fields_each(field, rp.inline_fields, rp.num_inline_fields, + rp.fields) + { + nfields++; + } nxt_http_fields_loop; + + if (rc != NXT_DONE || nfields != 24) { + nxt_log_alert(thr->log, "http parse fields reset test failed: " + "rc %i, %ui fields", rc, nfields); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_mp_destroy(mp); + + return ret; +} + + static nxt_int_t nxt_http_parse_test_run(nxt_http_request_parse_t *rp, nxt_str_t *request) { diff --git a/test/fake_upstream/README.md b/test/fake_upstream/README.md index b510d64a7..fe149113d 100644 --- a/test/fake_upstream/README.md +++ b/test/fake_upstream/README.md @@ -76,6 +76,7 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: | Port | Token | Mode (CLI) | Rust handler | pytest | |------|-------|-----------|--------------|--------| +| 7977 | — | — (plain-Python upstream in the test file) | — | `test_proxy_interim.py` (the proxy drops upstream 1xx responses and sends the final one) | | 7978 | — | — (plain-Python upstream in the test file) | — | `test_proxy_head.py` (bodyless upstream response: header block, with or without trailing bytes, then close — not a `fake_upstream` mode, reserved here so the 79xx block stays in one registry) | | 7979 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_keepalive_disabled` (duplicate upstream Content-Length disables downstream keepalive for a keep-alive client; complete body keeps its terminal chunk) — out of the 7983–7999 block below, which is exhausted; 7980–7982 are `fake_otlp`, so 7974–7979 is the free gap | | 7983 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_raw` (zero raw `Content-Length` occurrences on the wire, complete re-framed chunked body) | @@ -97,7 +98,8 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: > **7999 is not available here.** `test/test_proxy.py` and `test/test_proxy_chunked.py` > already bind `SERVER_PORT = 7999` for their own upstream, so this registry ends at -> 7998. New `fake_upstream` slots go to the 7974–7977 gap (7978–7979 are taken above, 7980–7982 are `fake_otlp`). +> 7998. New `fake_upstream` slots go to the 7974–7976 gap (7977 is taken by +> `test_proxy_interim.py` above, 7978–7979 are taken above, 7980–7982 are `fake_otlp`). A test pins its port as a module constant referencing this table: diff --git a/test/test_proxy_head.py b/test/test_proxy_head.py index 9be5b4e81..a5d233cca 100644 --- a/test/test_proxy_head.py +++ b/test/test_proxy_head.py @@ -12,9 +12,7 @@ Each case pipelines two requests in a single write: the bodyless one, then a plain GET. The second response arriving at all is the regression assertion. -1xx is out of scope here: an upstream 1xx is an interim response, and nothing -in the peer reader continues the exchange past it (that is pre-existing, see -nxt_h1p_peer_header_parse), so it is left on the path it already had. +1xx is tested in test_proxy_interim.py. These cases also drive both shapes the header read can take -- with and without body bytes alongside the header -- but they cannot assert that the diff --git a/test/test_proxy_interim.py b/test/test_proxy_interim.py new file mode 100644 index 000000000..20ce7dba3 --- /dev/null +++ b/test/test_proxy_interim.py @@ -0,0 +1,351 @@ +"""Proxied upstream 1xx interim responses (RFC 9110 Sect. 15.2). + +The proxy drops every 1xx except 101 and sends only the final response. +More than ten 1xx responses, or one bigger than the header buffer, give 502. + +Most cases send a second request (GET /ok) on the same connection. If its +response is correct, the first final response was not lost. Cases that end +with an error or 101 send one request only. +""" + +import select +import socket +import time + +import pytest + +from conftest import run_process +from unit.applications.proto import ApplicationProto +from unit.utils import waitforsocket + +client = ApplicationProto() + +# Reserved in test/fake_upstream/README.md's port registry. +UPSTREAM_PORT = 7977 + +FINAL = 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK' + +# A final response with its own field. +MARKED_FINAL = ( + 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\nX-Final: yes\r\n\r\nOK' +) + +# A typical 103. +INTERIM = 'HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n' + +# More than the 16 inline fields of the parser. +BIG_FIELD_COUNT = 20 + +# The only fields the final response may have. +FINAL_FIELDS = { + 'Content-Length', 'X-Final', 'Server', 'Date', 'Connection', + 'Transfer-Encoding', +} + +# A 103 bigger than half of the 64 KiB header buffer. Two of them fit only +# if the proxy frees the bytes of each dropped 1xx. +BIG_HINTS = 'HTTP/1.1 103 Early Hints\r\n' + ( + ''.join( + f'Link: ; rel=preload; as=style\r\n' + for i in range(700) + ) +) + '\r\n' + +assert 32 * 1024 < len(BIG_HINTS) < 64 * 1024, len(BIG_HINTS) + +# A 103 bigger than the header buffer. +OVERSIZED_HINTS = 'HTTP/1.1 103 Early Hints\r\n' + ( + ''.join(f'X-Pad-{i:02d}: {"a" * 1000}\r\n' for i in range(70)) +) + '\r\n' + +assert len(OVERSIZED_HINTS) > 64 * 1024, len(OVERSIZED_HINTS) + +# Upstream response for each target. A string is sent in one write. Each +# item of a list is sent separately, with a pause, so Unit reads it separately. +UPSTREAM_RESPONSES = { + # 1xx and final response in one read. + '/continue': 'HTTP/1.1 100 Continue\r\n\r\n' + FINAL, + # Final response in a later read. + '/hints': [ + 'HTTP/1.1 103 Early Hints\r\n' + 'Link: ; rel=preload; as=style\r\n\r\n', + FINAL, + ], + # Several 1xx responses in several reads. + '/multi': [ + 'HTTP/1.1 100 Continue\r\n\r\n', + 'HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n', + 'HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n' + + FINAL, + ], + '/big': BIG_HINTS + BIG_HINTS + FINAL, + # The empty line of the 1xx is split between two reads. + '/split': [INTERIM[:-1], '\n' + FINAL], + # Fields of the 1xx must not appear in the final response. + '/fields': ( + 'HTTP/1.1 103 Early Hints\r\n' + + ''.join(f'X-Early-{i:02d}: v{i}\r\n' for i in range(BIG_FIELD_COUNT)) + + 'Link: ; rel=preload\r\n' + + '\r\n' + + MARKED_FINAL + ), + # Framing fields on a 1xx (not allowed, RFC 9112, 6.1) are dropped too. + '/framing': ( + 'HTTP/1.1 103 Early Hints\r\nContent-Length: 3\r\n' + 'Connection: close\r\nLink: ; rel=preload\r\n\r\n' + FINAL + ), + # Exactly the cap, and one past it. + '/ten': INTERIM * 10 + FINAL, + '/eleven': INTERIM * 11 + FINAL, + # 101 changes the protocol, so it is the response. + '/switch': 'HTTP/1.1 101 Switching Protocols\r\n\r\n', + # A 1xx without a final response. + '/truncated': 'HTTP/1.1 100 Continue\r\n\r\n', + # A 1xx bigger than the header buffer. + '/oversized': OVERSIZED_HINTS + FINAL, + # The second request. + '/ok': FINAL, +} + + +def run_server(server_port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.bind(('127.0.0.1', server_port)) + sock.listen(10) + + def recv_until(conn, data, done): + while not done(data): + if not select.select([conn], [], [], 5)[0]: + break + + part = conn.recv(65536) + if not part: + break + + data += part + + return data + + while True: + conn, _ = sock.accept() + + data = recv_until(conn, b'', lambda d: b'\r\n\r\n' in d) + request = data.decode('utf-8', errors='ignore') + target = request.split(' ')[1] if ' ' in request else '' + + if target == '/expect': + # Read the body, send 100, then send how many bytes were read. + header, _, body = data.partition(b'\r\n\r\n') + length = 0 + + for line in header.split(b'\r\n'): + if line.lower().startswith(b'content-length:'): + length = int(line.split(b':', 1)[1]) + + body = recv_until(conn, body, lambda d: len(d) >= length) + + conn.sendall(b'HTTP/1.1 100 Continue\r\n\r\n') + time.sleep(0.1) + + reply = f'got {len(body)}'.encode() + conn.sendall( + b'HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n%s' + % (len(reply), reply) + ) + + else: + response = UPSTREAM_RESPONSES.get( + target, 'HTTP/1.1 500 Internal Server Error\r\n\r\n' + ) + + if isinstance(response, str): + response = [response] + + try: + for i, part in enumerate(response): + if i > 0: + time.sleep(0.1) + + conn.sendall(part.encode()) + + except OSError: + # Unit closed the connection before reading it all. + pass + + conn.close() + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + run_process(run_server, UPSTREAM_PORT) + waitforsocket(UPSTREAM_PORT) + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"action": {"proxy": f'http://127.0.0.1:{UPSTREAM_PORT}'}} + ], + } + ), 'interim proxy configuration' + + +def pipeline(first_request, sentinel=True): + """Send first_request, then GET /ok if sentinel is true. Read to EOF.""" + + request = first_request + + if sentinel: + request += ( + 'GET /ok HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + '\r\n' + ) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.settimeout(10) + + try: + sock.connect(('127.0.0.1', 8080)) + sock.sendall(request.encode()) + + data = b'' + while True: + try: + part = sock.recv(4096) + except socket.timeout: + break + + if not part: + break + + data += part + + finally: + sock.close() + + return data.decode('utf-8', errors='ignore') + + +def get(target): + return pipeline( + f'GET {target} HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + ) + + +def check(raw, body='OK'): + """Exactly two 200 responses, with the expected bodies, and no 1xx.""" + + assert 'HTTP/1.1 1' not in raw, f'1xx relayed to the client: {raw!r}' + + parts = raw.split('\r\n\r\n') + + assert len(parts) == 3, f'expected two responses, got {raw!r}' + + first, first_body_and_second, second_body = parts + + assert first.startswith('HTTP/1.1 200'), f'first status: {first!r}' + assert first_body_and_second.startswith( + body + ), f'first body: {first_body_and_second!r}' + assert ( + 'HTTP/1.1 200' in first_body_and_second + ), f'pipelined request lost: {first_body_and_second!r}' + assert second_body == 'OK', f'pipelined body: {second_body!r}' + + +def get_alone(target): + return pipeline( + f'GET {target} HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + '\r\n', + sentinel=False, + ) + + +@pytest.mark.parametrize( + 'target', + [ + '/ok', # control + '/continue', # interim and final in one read + '/hints', # final in a later read + '/multi', # several, split across reads + '/big', # fits only if the 1xx bytes are freed + '/split', # empty line of the 1xx in two reads + '/framing', # framing fields of the 1xx are dropped + '/ten', # exactly the cap + ], +) +def test_proxy_interim_consumed(target): + raw = get(target) + + check(raw) + + assert 'Content-Length: 3' not in raw, f'1xx framing relayed: {raw!r}' + + +def test_proxy_interim_expect_continue(): + """The upstream answers Expect with 100; the client gets only the final + response.""" + + body = 'x' * 4096 + + raw = pipeline( + 'POST /expect HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: keep-alive\r\n' + 'Expect: 100-continue\r\n' + f'Content-Length: {len(body)}\r\n' + '\r\n' + body + ) + + check(raw, body=f'got {len(body)}') + + +def test_proxy_interim_fields_not_leaked(): + """Fields of a 1xx, also those after the 16 inline fields, are dropped.""" + + raw = get('/fields') + + check(raw) + + lines = raw.split('\r\n\r\n')[0].split('\r\n') + names = [line.split(':', 1)[0] for line in lines[1:]] + + assert 'X-Final: yes' in raw, f'final field lost: {raw!r}' + assert not [ + name for name in names if name not in FINAL_FIELDS + ], f'interim fields leaked into the final response: {lines!r}' + + +def test_proxy_interim_101_is_the_response(): + """101 changes the protocol, so it is the response.""" + + raw = get_alone('/switch') + + assert raw.startswith( + 'HTTP/1.1 101 Switching Protocols' + ), f'101 not relayed: {raw!r}' + + +@pytest.mark.parametrize( + 'target', + [ + '/truncated', # upstream closes after a 1xx + '/eleven', # one past the cap + '/oversized', # a 1xx bigger than the buffer + ], +) +def test_proxy_interim_bad_gateway(target): + raw = get_alone(target) + + assert raw.startswith('HTTP/1.1 502'), f'expected 502: {raw[:200]!r}' + assert 'HTTP/1.1 200' not in raw, f'final relayed: {raw[:200]!r}' From 7df3fb4a6ab53a8684d2edcdae5b1a89b08f9df1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 21 Sep 2026 09:18:38 +0200 Subject: [PATCH 271/391] python: let the router's max_frame_size decide what an ASGI websocket accepts The ASGI websocket handler carried its own per-frame limit, nxt_py_asgi_ws_max_frame_size, a static 1 MiB that no configuration ever wrote. It duplicated the router's check at src/nxt_h1proto_websocket.c:360, which already applies the operator's settings/http/websocket/max_frame_size to every inbound frame before it crosses the port, except that the module's copy could not be raised. A 4 MiB message with max_frame_size at 32 MiB was closed with 1009 and the app saw RuntimeError("message too big"); no value of max_frame_size could lift that ceiling, because the key does not reach the worker at all. Worse, the counter it charged against, pending_frame_len, summed the fragments of a message, so the 1 MiB was really a per-message cap while the router checks per frame. At the default configuration a client fragmenting a larger message into small frames passed every router check and still got 1009. That is Autobahn 9.3.1-9.3.7 and 9.4.1-9.4.7, which this change turns green. pending_frame_len was also stale on the direct-delivery path. It is written only in suspend_frame, so when the final fragment arrives while the app is inside receive() the handler takes the pop_msg branch and the sum S of that message's non-final fragments stays behind. The defect is bounded, not monotonic: the next fragmented message's first data frame overwrites S rather than adding to it. But while only complete single-frame messages flow, each is charged against 1 MiB - S, so a 1 MiB message delivered in four 256 KiB fragments left the next single frame over 256 KiB refused with 1009. So delete the per-frame check and the field, and keep one cap, on pending_payload_len. It is now a #define rather than a mutable static that was never written, since its name invited confusion with the configuration key. What it bounds is the payload bytes of an incompletely delivered message, per connection: the router does no accounting across the fragments of one message and nothing else does either. A frame suspended while the app is not reading pins router shared memory in 16 KiB chunks, not worker heap -- nxt_unit_websocket_retain() returns early for an mmap-backed frame (src/nxt_unit.c:4155), and every frame above 12 bytes is mmap-backed. The surviving cap takes a floor of the frame's own payload_len, so a whole router-approved frame is always admitted into an empty queue and this module can never refuse what the configured max_frame_size allowed. The floor is also what makes the subtraction safe: limit >= payload_len by construction, where a plain CAP - pending wraps as soon as pending exceeds CAP, which the one admitted over-cap frame can do. At most one such frame is ever retained, so the steady-state retained payload per connection is max(10 MiB, max_frame_size), not the sum. CLOSE is exempt. The router already caps a control frame at 125 bytes, and refusing a CLOSE with 1009 would replace the ASGI websocket.disconnect the app is owed. Without the exemption a 12 MiB frame followed by a CLOSE reports 1009 where it should report 1000. The refusal path is deliberately unchanged: 1009 with RuntimeError("message too big") is the right reaction, the threshold was wrong. The cap is still not configurable and still not documented; an operator-visible message limit needs a channel to the worker and Node and Java parity, and is filed separately. Closes #429 Co-Authored-By: Claude Opus 5 --- src/python/nxt_python_asgi_websocket.c | 57 +++++++++++++------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/python/nxt_python_asgi_websocket.c b/src/python/nxt_python_asgi_websocket.c index 195864b4a..3590aaa0b 100644 --- a/src/python/nxt_python_asgi_websocket.c +++ b/src/python/nxt_python_asgi_websocket.c @@ -40,7 +40,6 @@ typedef struct { int state; nxt_queue_t pending_frames; uint64_t pending_payload_len; - uint64_t pending_frame_len; int pending_fins; } nxt_py_asgi_websocket_t; @@ -94,8 +93,17 @@ static PyTypeObject nxt_py_asgi_websocket_type = { .tp_methods = nxt_py_asgi_websocket_methods, }; -static uint64_t nxt_py_asgi_ws_max_frame_size = 1024 * 1024; -static uint64_t nxt_py_asgi_ws_max_buffer_size = 10 * 1024 * 1024; +/* + * Bounds the payload bytes a connection holds that the application has not + * yet received: the fragments of a message in progress, and complete messages + * waiting for receive(). The per-frame limit is the operator's + * settings/http/websocket/max_frame_size, applied by the router before the + * frame reaches this worker (the max_frame_size check in + * nxt_h1p_conn_ws_frame_header_read()); the router does no accounting across + * frames, which is what is bounded here. A suspended frame pins router shared + * memory, in 16 KiB chunks per frame, not worker heap. + */ +#define NXT_PY_ASGI_WS_MAX_BUFFER (10 * 1024 * 1024) int @@ -125,7 +133,6 @@ nxt_py_asgi_websocket_create(nxt_unit_request_info_t *req) ws->state = NXT_WS_INIT; nxt_queue_init(&ws->pending_frames); ws->pending_payload_len = 0; - ws->pending_frame_len = 0; ws->pending_fins = 0; } @@ -490,7 +497,7 @@ nxt_py_asgi_websocket_handler(nxt_unit_websocket_frame_t *frame) { uint8_t opcode; uint16_t status_code; - uint64_t rest; + uint64_t limit; PyObject *msg, *exc; nxt_py_asgi_websocket_t *ws; @@ -519,17 +526,23 @@ nxt_py_asgi_websocket_handler(nxt_unit_websocket_frame_t *frame) goto bad_state; } - rest = nxt_py_asgi_ws_max_frame_size - ws->pending_frame_len; - - if (nxt_slow_path(frame->payload_len > rest)) { - nxt_unit_websocket_done(frame); + /* + * The floor admits one whole router-approved frame into an empty queue, so + * this module never refuses a frame the configured max_frame_size allowed. + * It is also why the subtraction cannot wrap: limit >= payload_len by + * construction. CLOSE is exempt: the router caps a control frame at 125 + * bytes (nxt_h1p_conn_ws_frame_header_read()), and refusing a CLOSE would + * eat the ASGI disconnect message. + */ + limit = NXT_PY_ASGI_WS_MAX_BUFFER; - goto too_big; + if (frame->payload_len > limit) { + limit = frame->payload_len; } - rest = nxt_py_asgi_ws_max_buffer_size - ws->pending_payload_len; - - if (nxt_slow_path(frame->payload_len > rest)) { + if (nxt_slow_path(opcode != NXT_WEBSOCKET_OP_CLOSE + && ws->pending_payload_len > limit - frame->payload_len)) + { nxt_unit_websocket_done(frame); goto too_big; @@ -708,9 +721,9 @@ nxt_py_asgi_websocket_suspend_frame(nxt_unit_websocket_frame_t *frame) /* * Guard against uint64 wraparound across many fragmented frames; - * otherwise the eventual max_buffer_size check is bypassed. Run - * the check BEFORE inserting p into the pending_frames queue so a - * failure exit doesn't leak the suspended-frame slot. + * otherwise the NXT_PY_ASGI_WS_MAX_BUFFER check in the handler is + * bypassed. Run the check BEFORE inserting p into the pending_frames + * queue so a failure exit doesn't leak the suspended-frame slot. */ if (nxt_slow_path(frame->payload_len > UINT64_MAX - ws->pending_payload_len)) @@ -731,18 +744,6 @@ nxt_py_asgi_websocket_suspend_frame(nxt_unit_websocket_frame_t *frame) ws->pending_payload_len += frame->payload_len; ws->pending_fins += frame->header->fin; - - if (frame->header->fin) { - ws->pending_frame_len = 0; - - } else { - if (frame->header->opcode == NXT_WEBSOCKET_OP_CONT) { - ws->pending_frame_len += frame->payload_len; - - } else { - ws->pending_frame_len = frame->payload_len; - } - } } From ba346b738f7ea3c5d8e8eb10b6c68ee3afbf3473 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 21 Sep 2026 09:20:41 +0200 Subject: [PATCH 272/391] tests: pin the websocket message limits the ASGI module enforces Nothing in the default test leg reached the module's own 1009. The one test named for a limit, test_asgi_websockets_max_frame_size, exercises the router's check; the Autobahn 9.x block that would have caught this is skipped without --unsafe. That is how a per-message cap no configuration could reach survived for years. Seven cases, none of them behind --unsafe. Four are regression evidence, red on the parent commit and green here: a 2 MiB message in 32 fragments of 64 KiB at the default configuration; a 512 KiB frame after a fragmented message, which the stale per-frame counter charged against 1 MiB - 900 KiB; an 11 MiB single frame with max_frame_size at 16 MiB, the only coverage outside --unsafe of the floor that admits a whole router-approved frame; and ten frames of 2**20 - 14 bytes, 10,485,620 in total, just under the surviving 10 MiB. Three more pin properties of the new limit rather than the old bug. Eleven of those frames must still be refused, so the cap cannot be deleted by accident; a 1-byte CONT after an over-limit frame must be refused, so the queue can never hold two of them. Both of those also answer 1009 on the parent commit, for a different reason, so they are not before-and-after evidence. The third, a CLOSE after an over-limit frame, must report 1000 and is the only guard on the OP_CLOSE exemption; it is red on the parent too, but only because the parent refuses the 12 MiB frame itself, so what makes it evidence for the exemption is a control build carrying the floor without the exemption, which differs from this one in that one result and no other. check_mirror() names the close code rather than calling check_frame() directly, because check_frame() decodes a TEXT payload as UTF-8 and a refusal arrives as a CLOSE frame whose 2-byte body is not UTF-8: on the parent commit these would otherwise die with a UnicodeDecodeError instead of reporting 1009, which is exactly how test_asgi_websockets_9_1_1__9_6_6 reports it today. The fragmented-then-single case depends on the first message's final fragment reaching a waiting receive() rather than being suspended, which the mirror application makes near-certain; the comment says to rerun rather than add a sleep, and after the fix the counter it probes does not exist. Co-Authored-By: Claude Opus 5 --- test/test_asgi_websockets.py | 185 +++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index 191bc9dfa..65ee5153d 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -1453,6 +1453,191 @@ def test_asgi_websockets_10_1_1(): close_connection(sock) +def check_mirror(sock, opcode, payload): + """Read one message the mirror application echoed back. + + A refusal arrives as a CLOSE frame instead, and check_frame() decodes a + TEXT body as UTF-8, so a 2-byte close body would surface as a + UnicodeDecodeError rather than the close code. Name the code. + """ + frame = ws.frame_read(sock) + + assert frame['opcode'] != ws.OP_CLOSE, f"closed with {frame.get('code')}" + + check_frame(frame, True, opcode, payload) + + +def test_asgi_websockets_fragmented_message_over_1mb(): + client.load('websockets/mirror') + + _, sock, _ = ws.upgrade() + + # 2 MiB in 32 fragments of 64 KiB. Every frame is far below the default + # 1 MiB max_frame_size, so the router admits all of them; the module used + # to sum them against a private per-message 1 MiB of its own. + payload = '*' * 2 * 2**20 + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=64 * 2**10) + + check_mirror(sock, ws.OP_TEXT, payload) + + close_connection(sock) + + +def test_asgi_websockets_message_after_fragmented_message(): + client.load('websockets/mirror') + + _, sock, _ = ws.upgrade() + + # Three non-final 300 KiB fragments and a 100 KiB final one: under 1 MiB + # in total, so this message passed even the old cap. + payload = '*' * (3 * 300 * 2**10 + 100 * 2**10) + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=300 * 2**10) + + check_mirror(sock, ws.OP_TEXT, payload) + + # The old per-frame counter was never reset when the final fragment was + # delivered straight to a waiting receive(), so 900 KiB stayed on it and + # this 512 KiB frame was charged against 1 MiB - 900 KiB. That depends on + # the message above taking the direct-delivery branch, which the mirror + # application makes near-certain but does not guarantee: if this ever + # reports a 1009 on an unfixed build, rerun it, do not add a sleep. + payload = '*' * 512 * 2**10 + + ws.frame_write(sock, ws.OP_TEXT, payload) + + check_mirror(sock, ws.OP_TEXT, payload) + + close_connection(sock) + + +def test_asgi_websockets_frame_over_buffer_limit(): + client.load('websockets/mirror') + + assert 'success' in client.conf( + { + 'http': { + 'websocket': { + 'max_frame_size': 16777216, + 'keepalive_interval': 0, + } + } + }, + 'settings', + ), 'increase max_frame_size' + + _, sock, _ = ws.upgrade() + + # A single frame the configuration allows but which is larger than the + # module's own 10 MiB accumulation limit. The floor on that limit is what + # admits it; this is the only coverage of that branch outside --unsafe. + payload = '*' * 11 * 2**20 + + ws.frame_write(sock, ws.OP_TEXT, payload) + + check_mirror(sock, ws.OP_TEXT, payload) + + close_connection(sock) + + +def test_asgi_websockets_message_at_buffer_limit(): + client.load('websockets/mirror') + + _, sock, _ = ws.upgrade() + + # A masked frame with an 8-byte length carries 14 bytes of header, so this + # payload makes a frame of exactly the default 1 MiB max_frame_size. Ten + # of them are 10,485,620 bytes, just under the module's 10 MiB. + f_size = 2**20 - 14 + payload = '*' * (10 * f_size) + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=f_size) + + check_mirror(sock, ws.OP_TEXT, payload) + + close_connection(sock) + + +def test_asgi_websockets_message_over_buffer_limit(): + client.load('websockets/mirror') + + _, sock, _ = ws.upgrade() + + # One frame more than the test above: 11,534,182 bytes, over the 10 MiB. + # A property test, not a regression one -- an unfixed build also answers + # 1009 here, on the second frame rather than the eleventh. + f_size = 2**20 - 14 + payload = '*' * (11 * f_size) + + ws.message(sock, ws.OP_TEXT, payload, fragmention_size=f_size) + + check_close(sock, 1009) # 1009 - CLOSE_TOO_LARGE + + +def test_asgi_websockets_second_frame_over_buffer_limit(): + client.load('websockets/mirror') + + assert 'success' in client.conf( + { + 'http': { + 'websocket': { + 'max_frame_size': 33554432, + 'keepalive_interval': 0, + } + } + }, + 'settings', + ), 'increase max_frame_size' + + _, sock, _ = ws.upgrade() + + # The floor admits one over-limit frame into an empty queue and no more, + # so the connection holds at most one. Non-final, so it is suspended + # whatever the application is doing. Also a property test: an unfixed + # build refuses the 12 MiB frame itself. + ws.frame_write(sock, ws.OP_TEXT, '*' * 12 * 2**20, fin=False) + + # Without this pause the CONT frame can reach the application before the + # 12 MiB frame has been processed; on CI that failed with "Invalid state: + # attempt to process CONT frame". + time.sleep(2) + + ws.frame_write(sock, ws.OP_CONT, '*', fin=True) + + check_close(sock, 1009) # 1009 - CLOSE_TOO_LARGE + + +def test_asgi_websockets_close_after_frame_over_buffer_limit(): + client.load('websockets/mirror') + + assert 'success' in client.conf( + { + 'http': { + 'websocket': { + 'max_frame_size': 33554432, + 'keepalive_interval': 0, + } + } + }, + 'settings', + ), 'increase max_frame_size' + + _, sock, _ = ws.upgrade() + + ws.frame_write(sock, ws.OP_TEXT, '*' * 12 * 2**20, fin=False) + + # The same pause as above: let the 12 MiB frame be processed first. + time.sleep(2) + + # CLOSE is exempt from the accumulation limit: it is at most 125 bytes and + # refusing it would replace the disconnect the application is owed with a + # 1009. Drop the exemption and this returns 1009 instead of 1000. + ws.frame_write(sock, ws.OP_CLOSE, ws.serialize_close()) + + check_close(sock) + + # settings From c494bbe6d2d21b4b32a4a9d4e346cb7ffcbd4c7f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 21 Sep 2026 09:33:05 +0200 Subject: [PATCH 273/391] changelog: an ASGI websocket message is no longer capped at 1 MB Co-Authored-By: Claude Opus 5 --- CHANGES | 15 +++++++++++++++ docs/changes.xml | 17 +++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/CHANGES b/CHANGES index 6fb2dc96a..da6e7481c 100644 --- a/CHANGES +++ b/CHANGES @@ -445,6 +445,21 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 ReadOnlyBufferException instead of being sent. Both the blocking and the asynchronous remote were affected (#486). + *) Bugfix: the Python ASGI module refused a WebSocket message larger + than 1 MB whatever "max_frame_size" was set to. The module carried a + private 1 MB limit that no configuration ever wrote and charged the + fragments of a message against it, while the configured value is + checked by the router per frame: a 4 MB message was closed with 1009 + where "max_frame_size" was 32 MB, and at the default size a client + that split a larger message into small frames was refused as well. + Each frame is now limited by "max_frame_size" alone. The module + only limits a connection to 10 MB of payload that the application + has not received yet (fragments of a message in progress and + complete messages waiting for receive()); a single frame larger + than that is still accepted when nothing else is waiting. A + fragmented message also left a count behind that lowered the limit + applied to the next message. + Changes with FreeUnit 1.36.1 28 Aug 2026 diff --git a/docs/changes.xml b/docs/changes.xml index 2b48f85bf..62b557c15 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -586,6 +586,23 @@ asynchronous remote were affected (#486). + + +the Python ASGI module refused a WebSocket message larger than 1 MB whatever +"max_frame_size" was set to. The module carried a private 1 MB limit that no +configuration ever wrote and charged the fragments of a message against it, +while the configured value is checked by the router per frame: a 4 MB message +was closed with 1009 where "max_frame_size" was 32 MB, and at the default size +a client that split a larger message into small frames was refused as well. +Each frame is now limited by "max_frame_size" alone. The module only limits a +connection to 10 MB of payload that the application has not received yet +(fragments of a message in progress and complete messages waiting for +receive()); a single frame larger than that is still accepted when nothing else +is waiting. A fragmented message also left a count behind that lowered the +limit applied to the next message. + + + From 348c3b6b78eb07f3ff9a9495706a3e228b6eba6e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 16 Sep 2026 01:23:27 +0200 Subject: [PATCH 274/391] port: bound the queued messages that hold a descriptor port->messages is unbounded. Before a queued message owned the descriptors it names, a peer that stopped reading cost the sender memory. Now each queued message holds up to two descriptors of this process open for as long as it waits, so the same peer applies RLIMIT_NOFILE pressure as well. Bound that, and only that. A message with no descriptor still costs only memory, which is not what changed, and bounding the queue as a whole would change behaviour on every ordinary reply and every body fragment. So NXT_PORT_MAX_FD_MSGS caps the descriptor-carrying entries of one port, and the queue itself stays unbounded. A send that would go past the bound is refused with NXT_ERROR rather than dropped. That answer already means "nothing was consumed, the message is still yours" -- it is what nxt_port_msg_chk_insert() answers when the copy cannot be allocated -- so every caller that sends a descriptor already closes it on a failed send. Co-Authored-By: Claude Opus 5 (1M context) --- src/nxt_port.h | 34 +++ src/nxt_port_socket.c | 125 +++++++++- src/test/nxt_port_queued_fd_test.c | 356 +++++++++++++++++++++++++++++ 3 files changed, 510 insertions(+), 5 deletions(-) diff --git a/src/nxt_port.h b/src/nxt_port.h index 836598b01..2d3ac81b8 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -311,6 +311,30 @@ nxt_port_recv_msg_close_fds(nxt_port_recv_msg_t *msg) } +/* + * How many queued messages on one port may carry a file descriptor. + * + * port->messages itself stays unbounded. A message with no descriptor costs + * only memory, which was true before a queued message took ownership of what + * it names, and bounding it would change behaviour on paths that never hold a + * descriptor at all -- every ordinary reply and every request body fragment. + * + * A message that does carry one is different: it holds up to two descriptors + * of this process open for as long as it waits, so a peer that stops reading + * turns into RLIMIT_NOFILE pressure on the sender rather than memory pressure + * alone. That is the cost this bounds. + * + * The traffic being bounded is control-plane and one message per event: a new + * port, a process start, a listening socket, a certificate, a script, a shared + * memory segment. A port with 128 of them outstanding is not a busy port, it + * is a peer that has stopped reading, so the bound is far above any legitimate + * burst. At two descriptors an entry it caps one stalled port at 256 open + * descriptors, which leaves room for several of them under the 1024 soft + * RLIMIT_NOFILE that is still the common default. + */ +#define NXT_PORT_MAX_FD_MSGS 128 + + typedef struct nxt_app_s nxt_app_t; struct nxt_port_s { @@ -329,6 +353,16 @@ struct nxt_port_s { nxt_queue_t messages; /* of nxt_port_send_msg_t */ nxt_thread_mutex_t write_mutex; + /* + * How many entries of ->messages still carry a file descriptor. + * + * A queued message owns the descriptors it names, so each such entry + * holds up to two of this process's descriptors open for as long as it + * waits. The count exists to bound that; it is maintained under + * ->write_mutex and is described with the bound in src/nxt_port_socket.c. + */ + uint32_t fd_messages; + /* Maximum size of message part. */ uint32_t max_size; /* Maximum interleave of message parts. */ diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 081f15bd7..2f86118f6 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -13,7 +13,6 @@ #define NXT_PORT_MAX_ENQUEUE_BUF_SIZE \ (int) (NXT_PORT_QUEUE_MSG_SIZE - sizeof(nxt_port_msg_t)) - static nxt_bool_t nxt_port_can_enqueue_buf(nxt_buf_t *b); static uint8_t nxt_port_enqueue_buf(nxt_task_t *task, nxt_port_msg_t *pm, void *qbuf, nxt_buf_t *b); @@ -25,6 +24,11 @@ static nxt_int_t nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, nxt_bool_t *send_failed); static void nxt_port_write_handler(nxt_task_t *task, void *obj, void *data); static nxt_port_send_msg_t *nxt_port_msg_first(nxt_port_t *port); +nxt_inline nxt_bool_t nxt_port_msg_has_fd(const nxt_port_send_msg_t *msg); +nxt_inline void nxt_port_msg_fd_uncount_locked(nxt_port_t *port, + nxt_port_send_msg_t *msg); +static void nxt_port_msg_fd_uncount(nxt_port_t *port, + nxt_port_send_msg_t *msg); nxt_inline void nxt_port_msg_close_fd(nxt_port_send_msg_t *msg); nxt_inline void nxt_port_close_fds(nxt_fd_t *fd); static nxt_buf_t *nxt_port_buf_completion(nxt_task_t *task, @@ -472,7 +476,11 @@ static nxt_int_t nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, nxt_port_send_msg_t *msg) { - nxt_int_t res; + nxt_int_t res; + nxt_bool_t has_fd, over_bound; + + has_fd = nxt_port_msg_has_fd(msg); + over_bound = 0; nxt_thread_mutex_lock(&port->write_mutex); @@ -481,11 +489,30 @@ nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, { res = NXT_DECLINED; + } else if (nxt_slow_path(has_fd + && port->fd_messages >= NXT_PORT_MAX_FD_MSGS)) + { + /* + * Refuse rather than drop. NXT_ERROR from here says nothing of the + * message was consumed and it is still the caller's -- the same + * answer this function already gives when nxt_port_msg_alloc() + * fails, and the contract in src/nxt_port.h -- so every caller that + * sends a descriptor already has cleanup for it. + */ + + over_bound = 1; + res = NXT_ERROR; + } else { msg = nxt_port_msg_alloc(msg); if (nxt_fast_path(msg != NULL)) { nxt_queue_insert_tail(&port->messages, &msg->link); + + if (has_fd) { + port->fd_messages++; + } + nxt_port_use(task, port, 1); res = NXT_OK; @@ -496,6 +523,13 @@ nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, nxt_thread_mutex_unlock(&port->write_mutex); + if (nxt_slow_path(over_bound)) { + nxt_alert(task, "port{%d,%d} %d: %d queued messages already hold a " + "descriptor, refusing to queue another", + (int) port->pid, (int) port->id, port->socket.fd, + NXT_PORT_MAX_FD_MSGS); + } + return res; } @@ -541,7 +575,8 @@ nxt_port_msg_alloc(const nxt_port_send_msg_t *m) * * The dup holds the descriptor until the message goes out, so a peer * that stops reading holds this side's descriptor table open in - * proportion to what it was sent; port->messages has no bound. + * proportion to what it was sent. That is what NXT_PORT_MAX_FD_MSGS + * bounds; the callers of this function check it before they get here. */ if (!msg->close_fd && nxt_slow_path(nxt_port_msg_dup_fds(msg) != NXT_OK)) { @@ -913,6 +948,8 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, goto fail; } + nxt_port_msg_fd_uncount(port, msg); + nxt_port_msg_close_fd(msg); msg->buf = nxt_port_buf_completion(task, wq, msg->buf, plain_size, @@ -1173,6 +1210,8 @@ nxt_port_socket_cancel(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, break; } + nxt_port_msg_fd_uncount_locked(port, msg); + nxt_queue_remove(&msg->link); msg->link.next = NULL; @@ -1246,6 +1285,51 @@ nxt_port_msg_first(nxt_port_t *port) } +nxt_inline nxt_bool_t +nxt_port_msg_has_fd(const nxt_port_send_msg_t *msg) +{ + return msg->fd[0] != -1 || msg->fd[1] != -1; +} + + +/* + * Stop counting a queued message against NXT_PORT_MAX_FD_MSGS. + * + * The count follows one invariant: an entry of port->messages is counted + * exactly while it carries a descriptor. So this runs at the two moments + * that end it -- the message leaves the queue, or it is still queued but has + * just had its descriptors sent -- and must run before whatever clears + * msg->fd[], since that is what it reads. msg->link.next is the queue's own + * "is it in the list" marker, which keeps the inline stack copy in + * nxt_port_socket_write2() out of the count. + * + * The _locked form is for the callers that already hold port->write_mutex. + */ + +nxt_inline void +nxt_port_msg_fd_uncount_locked(nxt_port_t *port, nxt_port_send_msg_t *msg) +{ + if (msg->link.next != NULL && nxt_port_msg_has_fd(msg)) { + port->fd_messages--; + } +} + + +static void +nxt_port_msg_fd_uncount(nxt_port_t *port, nxt_port_send_msg_t *msg) +{ + if (msg->link.next == NULL || !nxt_port_msg_has_fd(msg)) { + return; + } + + nxt_thread_mutex_lock(&port->write_mutex); + + port->fd_messages--; + + nxt_thread_mutex_unlock(&port->write_mutex); +} + + nxt_inline void nxt_port_msg_close_fd(nxt_port_send_msg_t *msg) { @@ -1336,21 +1420,50 @@ nxt_port_buf_completion(nxt_task_t *task, nxt_work_queue_t *wq, nxt_buf_t *b, } +/* + * The bound applies here as well: this is the other way a descriptor-carrying + * message enters port->messages, when an inline first fragment hits EAGAIN and + * has to be held for a later attempt. Answering NULL is what an allocation + * failure already answers, and nxt_port_write_msgs() turns both into the same + * "nothing was consumed" NXT_ERROR for a first fragment. + */ + static nxt_port_send_msg_t * nxt_port_msg_insert_tail(nxt_port_t *port, nxt_port_send_msg_t *msg) { + nxt_bool_t has_fd; + + has_fd = nxt_port_msg_has_fd(msg); + + nxt_thread_mutex_lock(&port->write_mutex); + + if (nxt_slow_path(has_fd && port->fd_messages >= NXT_PORT_MAX_FD_MSGS)) { + nxt_thread_mutex_unlock(&port->write_mutex); + + nxt_thread_log_alert("port{%d,%d}: %d queued messages already hold a " + "descriptor, refusing to queue another", + (int) port->pid, (int) port->id, + NXT_PORT_MAX_FD_MSGS); + + return NULL; + } + if (msg->allocated == 0) { msg = nxt_port_msg_alloc(msg); if (nxt_slow_path(msg == NULL)) { + nxt_thread_mutex_unlock(&port->write_mutex); + return NULL; } } - nxt_thread_mutex_lock(&port->write_mutex); - nxt_queue_insert_tail(&port->messages, &msg->link); + if (has_fd) { + port->fd_messages++; + } + nxt_thread_mutex_unlock(&port->write_mutex); return msg; @@ -2202,6 +2315,8 @@ nxt_port_error_handler(nxt_task_t *task, void *obj, void *data) nxt_queue_each(msg, &port->messages, nxt_port_send_msg_t, link) { + nxt_port_msg_fd_uncount_locked(port, msg); + nxt_port_msg_close_fd(msg); for (b = msg->buf; b != NULL; b = next) { diff --git a/src/test/nxt_port_queued_fd_test.c b/src/test/nxt_port_queued_fd_test.c index 89db817d7..2c767510f 100644 --- a/src/test/nxt_port_queued_fd_test.c +++ b/src/test/nxt_port_queued_fd_test.c @@ -21,6 +21,14 @@ * reopen something else at the same number, then let the write handler * send. The peer must receive the descriptor that was queued, not the * one that took its number. + * + * A third leg covers the bound that ownership made necessary + * (NXT_PORT_MAX_FD_MSGS, freeunitorg/freeunit#394). A peer that stops + * reading holds two of this process's descriptors per queued message, so + * the number of descriptor-carrying entries is capped. The leg fills a + * port to the cap, checks that the next descriptor-carrying send is + * refused with nothing consumed, that a send with no descriptor is still + * accepted, and that nothing is leaked either way. */ #include @@ -32,10 +40,12 @@ #include #include #include +#include static nxt_int_t nxt_port_queued_fd_test_owned(nxt_thread_t *thr); static nxt_int_t nxt_port_queued_fd_test_reused(nxt_thread_t *thr); +static nxt_int_t nxt_port_queued_fd_test_bounded(nxt_thread_t *thr); static nxt_port_t *nxt_port_queued_fd_test_port(nxt_task_t *task, nxt_event_engine_t *engine); static void nxt_port_queued_fd_test_stub(nxt_event_engine_t *engine, @@ -46,6 +56,8 @@ static void nxt_port_queued_fd_test_drain_wq(nxt_work_queue_t *wq); static nxt_bool_t nxt_port_queued_fd_test_same_file(nxt_fd_t a, const struct stat *b); static nxt_fd_t nxt_port_queued_fd_test_recv_fd(nxt_fd_t sock); +static nxt_uint_t nxt_port_queued_fd_test_open_fds(void); +static nxt_uint_t nxt_port_queued_fd_test_queued(nxt_port_t *port); static nxt_uint_t nxt_port_queued_fd_test_completions; @@ -65,6 +77,10 @@ nxt_port_queued_fd_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_queued_fd_test_bounded(thr) != NXT_OK) { + return NXT_ERROR; + } + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "port queued fd test passed"); @@ -502,6 +518,346 @@ nxt_port_queued_fd_test_reused(nxt_thread_t *thr) } +/* + * The bound on descriptor-carrying entries. + * + * The port is not writable, so every send queues, and each queued copy dups + * what it was given: the open descriptor count rises by one per message and + * is the measurement the leg makes. Nothing here reaches inside the port to + * read a counter -- the bound is observed the way a caller observes it, from + * the answer to a send and from what the process still holds open. + */ + +static nxt_int_t +nxt_port_queued_fd_test_bounded(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_uint_t i, base, filled; + nxt_fd_t fd, got, pair[2]; + nxt_buf_t *buf; + nxt_int_t ret; + struct stat st; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine, *saved_engine; + nxt_event_interface_t stub; + + task = thr->task; + task->thread = thr; + + ret = NXT_ERROR; + fd = -1; + got = -1; + pair[0] = -1; + pair[1] = -1; + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_queued_fd_test_stub; + stub.block_write = nxt_port_queued_fd_test_stub; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + thr->engine = saved_engine; + return NXT_ERROR; + } + + port = nxt_port_queued_fd_test_port(task, &engine); + if (nxt_slow_path(port == NULL)) { + nxt_mp_destroy(mp); + thr->engine = saved_engine; + return NXT_ERROR; + } + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: socketpair failed"); + goto done; + } + + if (nxt_slow_path(fcntl(pair[1], F_SETFL, O_NONBLOCK) == -1)) { + goto done; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + + nxt_port_write_enable(task, port); + + port->socket.write_ready = 0; + port->socket.write = NXT_EVENT_INACTIVE; + + fd = open("/dev/null", O_RDONLY); + if (fd == -1 || fstat(fd, &st) != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: failed to open /dev/null"); + goto done; + } + + nxt_port_queued_fd_test_completions = 0; + + base = nxt_port_queued_fd_test_open_fds(); + + /* Fill the queue to the bound. Every one of these must be taken. */ + + for (i = 0; i < NXT_PORT_MAX_FD_MSGS; i++) { + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + i, 0, NULL) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: message %ui of %d was " + "refused below the bound", i, NXT_PORT_MAX_FD_MSGS); + goto done; + } + } + + filled = nxt_port_queued_fd_test_queued(port); + + if (filled != NXT_PORT_MAX_FD_MSGS) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui messages queued, expected %d", + filled, NXT_PORT_MAX_FD_MSGS); + goto done; + } + + if (nxt_port_queued_fd_test_open_fds() != base + NXT_PORT_MAX_FD_MSGS) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui descriptors open after the " + "fill, expected %ui", + nxt_port_queued_fd_test_open_fds(), + base + NXT_PORT_MAX_FD_MSGS); + goto done; + } + + /* + * One more, with a payload. It must be refused, and refused whole: the + * descriptor is still the caller's and open, the buffer is still the + * caller's and uncompleted, and the queue is as it was. + */ + + buf = nxt_buf_mem_alloc(mp, 1, 0); + if (nxt_slow_path(buf == NULL)) { + goto done; + } + + buf->completion_handler = nxt_port_queued_fd_test_completion; + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + NXT_PORT_MAX_FD_MSGS, 0, buf) + != NXT_ERROR) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: a message past the bound of %d " + "was accepted", NXT_PORT_MAX_FD_MSGS); + goto done; + } + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_queued_fd_test_completions != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the refused message completed " + "%ui buffers, expected 0", + nxt_port_queued_fd_test_completions); + goto done; + } + + if (!nxt_test_fd_is_open(fd)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the refused message closed the " + "caller's descriptor"); + fd = -1; + goto done; + } + + if (nxt_port_queued_fd_test_queued(port) != NXT_PORT_MAX_FD_MSGS + || nxt_port_queued_fd_test_open_fds() != base + NXT_PORT_MAX_FD_MSGS) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the refused message changed the " + "queue"); + goto done; + } + + /* + * A message with no descriptor is not what the bound is about, and is + * still taken on the same full port. + */ + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_DATA, -1, -1, + NXT_PORT_MAX_FD_MSGS + 1, 0, buf) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: a message with no descriptor was " + "refused on a port at the bound"); + goto done; + } + + if (nxt_port_queued_fd_test_queued(port) != NXT_PORT_MAX_FD_MSGS + 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the message with no descriptor " + "was not queued"); + goto done; + } + + /* + * Draining is what releases the bound. The peer gets what was queued + * below it, and the duplicates go with the messages: the descriptor + * count comes back to where it started. + */ + + port->socket.write_ready = 1; + + port->socket.write_handler(task, &port->socket, NULL); + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + got = nxt_port_queued_fd_test_recv_fd(pair[0]); + + if (got == -1 || !nxt_port_queued_fd_test_same_file(got, &st)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the peer did not receive the " + "first message queued below the bound"); + goto done; + } + + nxt_fd_close(got); + got = -1; + + /* Whatever the socket would not take is dropped the ordinary way. */ + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_port_test_run_error_handler(task, port); + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + } + + while ((got = nxt_port_queued_fd_test_recv_fd(pair[0])) != -1) { + nxt_fd_close(got); + } + + if (nxt_port_queued_fd_test_open_fds() != base) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui descriptors open after the " + "queue drained, expected the %ui it started with", + nxt_port_queued_fd_test_open_fds(), base); + goto done; + } + + /* And the port takes descriptor-carrying messages again. */ + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + 0, 0, NULL) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the drained port still refuses a " + "descriptor"); + goto done; + } + + ret = NXT_OK; + +done: + + if (fd != -1 && nxt_test_fd_is_open(fd)) { + nxt_fd_close(fd); + } + + if (got != -1) { + nxt_fd_close(got); + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_port_test_run_error_handler(task, port); + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + if (pair[0] != -1) { + nxt_fd_close(pair[0]); + } + + if (pair[1] != -1) { + nxt_fd_close(pair[1]); + } + + nxt_port_use(task, port, -1); + nxt_mp_destroy(mp); + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + thr->engine = saved_engine; + + return ret; +} + + +/* + * How many descriptors this process holds open. Counted by probing numbers + * rather than by reading /proc, which is not there on every platform the C + * suite builds on. + */ + +static nxt_uint_t +nxt_port_queued_fd_test_open_fds(void) +{ + nxt_fd_t fd; + nxt_uint_t n, limit; + struct rlimit rlmt; + + limit = 4096; + + if (getrlimit(RLIMIT_NOFILE, &rlmt) == 0 + && rlmt.rlim_cur != RLIM_INFINITY + && rlmt.rlim_cur < limit) + { + limit = rlmt.rlim_cur; + } + + n = 0; + + for (fd = 0; fd < (nxt_fd_t) limit; fd++) { + if (nxt_test_fd_is_open(fd)) { + n++; + } + } + + return n; +} + + +static nxt_uint_t +nxt_port_queued_fd_test_queued(nxt_port_t *port) +{ + nxt_uint_t n; + nxt_queue_link_t *lnk; + + n = 0; + + for (lnk = nxt_queue_first(&port->messages); + lnk != nxt_queue_tail(&port->messages); + lnk = nxt_queue_next(lnk)) + { + n++; + } + + return n; +} + + static void nxt_port_queued_fd_test_completion(nxt_task_t *task, void *obj, void *data) { From 4f81c407eb6a413101da84e933fcde8aab4030b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 22:23:24 +0000 Subject: [PATCH 275/391] fix(port): log a refused descriptor once per stall and harden the count A port at NXT_PORT_MAX_FD_MSGS stays there for as long as its peer does not read, and every refused send logged another alert. The port now remembers that it is refusing (->fd_refusing, under ->write_mutex): the first refusal logs, the next accepted descriptor re-arms the log. nxt_port_msg_fd_uncount() tested the message outside the mutex and then decremented without testing again. nxt_port_socket_cancel() can take the same message out of the queue between the two, and one extra decrement wraps the uint32_t count, after which the port refuses every descriptor for good. The test is now repeated under the mutex, and the decrement stops at zero. The bound leg now also covers the cancel decrement and the log-once rule, and is skipped with a notice when RLIMIT_NOFILE is too low to hold the fill, instead of failing on a dup that ran out of descriptors. The blank line the first commit dropped after NXT_PORT_MAX_ENQUEUE_BUF_SIZE is back. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- src/nxt_port.h | 7 ++ src/nxt_port_socket.c | 43 +++++++-- src/test/nxt_port_queued_fd_test.c | 141 +++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 8 deletions(-) diff --git a/src/nxt_port.h b/src/nxt_port.h index 2d3ac81b8..c2be2609d 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -360,8 +360,15 @@ struct nxt_port_s { * holds up to two of this process's descriptors open for as long as it * waits. The count exists to bound that; it is maintained under * ->write_mutex and is described with the bound in src/nxt_port_socket.c. + * + * ->fd_refusing is set by the first send the bound refuses and cleared + * by the next descriptor it takes, also under ->write_mutex. A peer + * that stops reading keeps the port at the bound for as long as it + * stalls, so the refusal is logged once when the port gets there rather + * than once for every send. */ uint32_t fd_messages; + uint8_t fd_refusing; /* Maximum size of message part. */ uint32_t max_size; diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 2f86118f6..ddf718fb6 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -13,6 +13,7 @@ #define NXT_PORT_MAX_ENQUEUE_BUF_SIZE \ (int) (NXT_PORT_QUEUE_MSG_SIZE - sizeof(nxt_port_msg_t)) + static nxt_bool_t nxt_port_can_enqueue_buf(nxt_buf_t *b); static uint8_t nxt_port_enqueue_buf(nxt_task_t *task, nxt_port_msg_t *pm, void *qbuf, nxt_buf_t *b); @@ -498,9 +499,12 @@ nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, * answer this function already gives when nxt_port_msg_alloc() * fails, and the contract in src/nxt_port.h -- so every caller that * sends a descriptor already has cleanup for it. + * + * Logged once per stall, not once per refusal: see ->fd_refusing. */ - over_bound = 1; + over_bound = !port->fd_refusing; + port->fd_refusing = 1; res = NXT_ERROR; } else { @@ -511,6 +515,7 @@ nxt_port_msg_chk_insert(nxt_task_t *task, nxt_port_t *port, if (has_fd) { port->fd_messages++; + port->fd_refusing = 0; } nxt_port_use(task, port, 1); @@ -1309,12 +1314,28 @@ nxt_port_msg_has_fd(const nxt_port_send_msg_t *msg) nxt_inline void nxt_port_msg_fd_uncount_locked(nxt_port_t *port, nxt_port_send_msg_t *msg) { - if (msg->link.next != NULL && nxt_port_msg_has_fd(msg)) { + /* + * The count is never below zero by the invariant, so the test on it + * only guards against a bug elsewhere. It is cheap, and a wrap is + * worse than the leak the bound fixes: at UINT32_MAX the port would + * refuse every descriptor for the rest of its life. + */ + + if (msg->link.next != NULL && nxt_port_msg_has_fd(msg) + && nxt_fast_path(port->fd_messages != 0)) + { port->fd_messages--; } } +/* + * The test before the lock only spares the mutex for the common message + * with no descriptor. It is repeated under the lock, because + * nxt_port_socket_cancel() can take the same message out of the queue, and + * uncount it, between the two. + */ + static void nxt_port_msg_fd_uncount(nxt_port_t *port, nxt_port_send_msg_t *msg) { @@ -1324,7 +1345,7 @@ nxt_port_msg_fd_uncount(nxt_port_t *port, nxt_port_send_msg_t *msg) nxt_thread_mutex_lock(&port->write_mutex); - port->fd_messages--; + nxt_port_msg_fd_uncount_locked(port, msg); nxt_thread_mutex_unlock(&port->write_mutex); } @@ -1431,19 +1452,24 @@ nxt_port_buf_completion(nxt_task_t *task, nxt_work_queue_t *wq, nxt_buf_t *b, static nxt_port_send_msg_t * nxt_port_msg_insert_tail(nxt_port_t *port, nxt_port_send_msg_t *msg) { - nxt_bool_t has_fd; + nxt_bool_t has_fd, log; has_fd = nxt_port_msg_has_fd(msg); nxt_thread_mutex_lock(&port->write_mutex); if (nxt_slow_path(has_fd && port->fd_messages >= NXT_PORT_MAX_FD_MSGS)) { + log = !port->fd_refusing; + port->fd_refusing = 1; + nxt_thread_mutex_unlock(&port->write_mutex); - nxt_thread_log_alert("port{%d,%d}: %d queued messages already hold a " - "descriptor, refusing to queue another", - (int) port->pid, (int) port->id, - NXT_PORT_MAX_FD_MSGS); + if (log) { + nxt_thread_log_alert("port{%d,%d}: %d queued messages already " + "hold a descriptor, refusing to queue " + "another", (int) port->pid, (int) port->id, + NXT_PORT_MAX_FD_MSGS); + } return NULL; } @@ -1462,6 +1488,7 @@ nxt_port_msg_insert_tail(nxt_port_t *port, nxt_port_send_msg_t *msg) if (has_fd) { port->fd_messages++; + port->fd_refusing = 0; } nxt_thread_mutex_unlock(&port->write_mutex); diff --git a/src/test/nxt_port_queued_fd_test.c b/src/test/nxt_port_queued_fd_test.c index 2c767510f..383a5b042 100644 --- a/src/test/nxt_port_queued_fd_test.c +++ b/src/test/nxt_port_queued_fd_test.c @@ -58,9 +58,13 @@ static nxt_bool_t nxt_port_queued_fd_test_same_file(nxt_fd_t a, static nxt_fd_t nxt_port_queued_fd_test_recv_fd(nxt_fd_t sock); static nxt_uint_t nxt_port_queued_fd_test_open_fds(void); static nxt_uint_t nxt_port_queued_fd_test_queued(nxt_port_t *port); +static void nxt_cdecl nxt_port_queued_fd_test_log(nxt_uint_t level, + nxt_log_t *log, const char *fmt, ...); static nxt_uint_t nxt_port_queued_fd_test_completions; +static nxt_uint_t nxt_port_queued_fd_test_alerts; +static nxt_log_t *nxt_port_queued_fd_test_saved_log; nxt_int_t @@ -536,7 +540,9 @@ nxt_port_queued_fd_test_bounded(nxt_thread_t *thr) nxt_fd_t fd, got, pair[2]; nxt_buf_t *buf; nxt_int_t ret; + nxt_log_t log, *saved_log; struct stat st; + struct rlimit rlmt; nxt_task_t *task; nxt_port_t *port; nxt_event_engine_t engine, *saved_engine; @@ -545,6 +551,28 @@ nxt_port_queued_fd_test_bounded(nxt_thread_t *thr) task = thr->task; task->thread = thr; + /* + * The fill holds NXT_PORT_MAX_FD_MSGS duplicates open at once, and the + * leg needs a few more of its own. Under a lower RLIMIT_NOFILE a dup + * fails before the bound is reached, which says nothing about the + * bound, so the leg is skipped rather than failed. + */ + + if (getrlimit(RLIMIT_NOFILE, &rlmt) == 0 + && rlmt.rlim_cur != RLIM_INFINITY + && rlmt.rlim_cur < nxt_port_queued_fd_test_open_fds() + + NXT_PORT_MAX_FD_MSGS + 16) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: bound leg skipped, " + "RLIMIT_NOFILE %d is too low for %d queued " + "descriptors", (int) rlmt.rlim_cur, + NXT_PORT_MAX_FD_MSGS); + return NXT_OK; + } + + saved_log = task->log; + ret = NXT_ERROR; fd = -1; got = -1; @@ -652,6 +680,19 @@ nxt_port_queued_fd_test_bounded(nxt_thread_t *thr) buf->completion_handler = nxt_port_queued_fd_test_completion; + /* + * The refusals below are counted by the alerts they log: a stalled peer + * keeps the port at the bound, so only the first one may log. + */ + + log = *saved_log; + log.handler = nxt_port_queued_fd_test_log; + + nxt_port_queued_fd_test_saved_log = saved_log; + nxt_port_queued_fd_test_alerts = 0; + + task->log = &log; + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, NXT_PORT_MAX_FD_MSGS, 0, buf) != NXT_ERROR) @@ -662,6 +703,23 @@ nxt_port_queued_fd_test_bounded(nxt_thread_t *thr) goto done; } + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + NXT_PORT_MAX_FD_MSGS, 0, NULL) + != NXT_ERROR) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: a second message past the bound " + "was accepted"); + goto done; + } + + if (nxt_port_queued_fd_test_alerts != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: two refusals in a row logged %ui " + "alerts, expected 1", nxt_port_queued_fd_test_alerts); + goto done; + } + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); if (nxt_port_queued_fd_test_completions != 0) { @@ -711,6 +769,57 @@ nxt_port_queued_fd_test_bounded(nxt_thread_t *thr) goto done; } + /* + * Cancelling a queued message releases its place under the bound as + * well: the port takes one more descriptor, and refusing the one after + * that is a new stall, so it logs again. + */ + + if (nxt_port_socket_cancel(task, port, NXT_PORT_MSG_NEW_PORT, 0, 0, NULL) + != NXT_PORT_MSG_CANCELLED) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the first queued message could " + "not be cancelled"); + goto done; + } + + nxt_port_queued_fd_test_drain_wq(&engine.fast_work_queue); + + if (nxt_port_queued_fd_test_open_fds() != base + NXT_PORT_MAX_FD_MSGS - 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: %ui descriptors open after the " + "cancel, expected %ui", + nxt_port_queued_fd_test_open_fds(), + base + NXT_PORT_MAX_FD_MSGS - 1); + goto done; + } + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + NXT_PORT_MAX_FD_MSGS + 2, 0, NULL) + != NXT_OK) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: a cancelled message did not " + "release its place under the bound"); + goto done; + } + + if (nxt_port_socket_write2(task, port, NXT_PORT_MSG_NEW_PORT, fd, -1, + NXT_PORT_MAX_FD_MSGS + 3, 0, NULL) + != NXT_ERROR + || nxt_port_queued_fd_test_alerts != 2) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port queued fd test: the port at the bound again did " + "not refuse and log once more (%ui alerts)", + nxt_port_queued_fd_test_alerts); + goto done; + } + + task->log = saved_log; + /* * Draining is what releases the bound. The peer gets what was queued * below it, and the duplicates go with the messages: the descriptor @@ -770,6 +879,8 @@ nxt_port_queued_fd_test_bounded(nxt_thread_t *thr) done: + task->log = saved_log; + if (fd != -1 && nxt_test_fd_is_open(fd)) { nxt_fd_close(fd); } @@ -839,6 +950,36 @@ nxt_port_queued_fd_test_open_fds(void) } +/* + * Counts the alerts and passes every line on to the log the leg replaced, + * so the refusal still shows in the output. + */ + +static void nxt_cdecl +nxt_port_queued_fd_test_log(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...) +{ + u_char *p; + va_list args; + nxt_log_t *saved; + u_char msg[NXT_MAX_ERROR_STR]; + + if (level == NXT_LOG_ALERT) { + nxt_port_queued_fd_test_alerts++; + } + + va_start(args, fmt); + p = nxt_vsprintf(msg, msg + NXT_MAX_ERROR_STR - 1, fmt, args); + va_end(args); + + *p = '\0'; + + saved = nxt_port_queued_fd_test_saved_log; + + saved->handler(level, saved, "%s", msg); +} + + static nxt_uint_t nxt_port_queued_fd_test_queued(nxt_port_t *port) { From c930359bb17532cfe4a804c86719954cf9713987 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 19 Sep 2026 10:44:52 +0200 Subject: [PATCH 276/391] router: count the application's detached work instead of latching a flag libunit tracks detached work per context, the router keys it per process, and the START and FINISH edges carry neither a request nor a context id. A worker running several contexts at once -- python, ruby and java all do -- could therefore have the first context's FINISH hand a still-busy process back to the idle economy: the port returns to the idle queues, the reaper may take it, and its slot returns to "processes": {"max"} while the other contexts are still executing. Make port->detached_app a uint32_t count, the way port->detached_router already is. START raises it, FINISH lowers it, and the aggregate port->detached, app->detached_processes and the application reference move only on the transitions that matter: the first reason taken and the last one settled. /status keeps reporting the processes and nxt_router_free_app() still sees zero. A START at UINT32_MAX saturates with an alert rather than wrapping to "no detached work", and a FINISH at zero -- a worker whose death has already settled the count, or a forged edge -- alerts and changes nothing instead of underflowing. nxt_router_app_port_close() settles whatever the count holds in one go, as it already did. The wire format is unchanged: the same two one-byte edges on the same message type, so old and new workers interoperate in either direction. nxt_port.h states the protocol contract next to nxt_port_detached_t: the edges are a balanced pair, they name no request or context, and the payload may grow. nxt_unit.h says libunit counts per context while the router counts per worker. nxt_router_detached_test.c becomes table-driven over 10 edges, adding two starts against one finish, the second finish that releases the worker, and an unmatched finish. Co-Authored-By: Claude Opus 5 --- src/nxt_port.h | 19 +++- src/nxt_router.c | 113 ++++++++++++++++-------- src/nxt_unit.c | 12 +-- src/nxt_unit.h | 9 +- src/test/nxt_router_detached_test.c | 132 +++++++++++++++++++++------- 5 files changed, 200 insertions(+), 85 deletions(-) diff --git a/src/nxt_port.h b/src/nxt_port.h index c2be2609d..72d94e3a3 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -201,6 +201,13 @@ typedef enum { * header field by field, and nxt_port_socket_write() ORs into ->last. A * new type is bounds-checked on both sides instead, so an older peer * refuses the message rather than misreading a flag. + * + * The two edges are a balanced pair: one start and one finish per unit of + * detached work. The payload names neither a request nor a context, so the + * router can only count the edges per process; it takes the worker out of + * the idle economy on the first start and gives it back on the last finish. + * The payload may grow later, and the handler requires at least one byte and + * reads the first, so an older sender stays readable. */ typedef enum { NXT_PORT_DETACHED_START = 0, @@ -392,11 +399,15 @@ struct nxt_port_s { uint8_t detached; /* - * The application reported detached work of its own. Its FINISH edge is - * what ends that; until then the port stays out of the idle economy even - * with no request left. + * How many units of detached work the application reported of its own: + * one for each START edge whose FINISH edge has not arrived. Until the + * last of them does, the port stays out of the idle economy even with no + * request left. A count rather than a flag because the edges carry no + * context id and a worker may run several contexts -- see + * nxt_port_detached_t above -- so the first context's FINISH must not + * speak for the rest. */ - uint8_t detached_app; + uint32_t detached_app; /* * The router put the port in the detached state itself: one for each diff --git a/src/nxt_router.c b/src/nxt_router.c index 43568fe49..c948e13b2 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -7029,7 +7029,10 @@ nxt_router_app_port_close(nxt_task_t *task, nxt_port_t *port) * after the response went out, or a kill. This is the only path that * runs for it then, so the count has to be settled here or the * application never reaches the zero nxt_router_free_app() asserts on. - * The matching application reference is dropped below, outside the lock. + * The process is gone, so every unit of work it was running goes with + * it, however many the counts hold. The matching application + * reference, the one the first reason took, is dropped below, outside + * the lock. */ detached = port->detached; @@ -8019,16 +8022,20 @@ nxt_router_http_request_release(nxt_task_t *task, void *obj, void *data) * field is never cleared, and the main port can be released while a sibling * port of the same process is still registered. * - * Either edge may arrive with the port already in the state being asked - * for, and both then do nothing: a start edge is idempotent because several - * contexts of one worker can be detached at once, and a finish edge because - * the worker's death has already settled the state. + * The application's edges are counted, not latched. Several contexts of one + * worker can run detached work at once and the edge names none of them, so + * the port leaves the idle economy on the first start and rejoins it on the + * last finish, once the router has no abandoned request of its own left + * either. app->detached_processes counts processes, so it moves on those two + * transitions only. A finish with nothing to match -- a worker whose death + * has already settled the count, or a forged one -- changes nothing. */ static void nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) { int drop; + const char *alert; nxt_app_t *app; nxt_port_t *port; nxt_bool_t changed, start_process, adjust_idle_timer; @@ -8084,6 +8091,7 @@ nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) nxt_debug(task, "app '%V' port %PI:%d detached state %d", &app->name, port->pid, port->id, (int) state); + alert = NULL; changed = 0; start_process = 0; adjust_idle_timer = 0; @@ -8093,41 +8101,64 @@ nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) if (state == NXT_PORT_DETACHED_START) { /* - * A reason of the application's own, held beside the router's count - * of requests it gave up on: neither clear may drop the other. + * A reason of the application's own, counted beside the router's + * count of requests it gave up on: neither may settle the other. + * + * Saturate rather than wrap. The count is bounded by the contexts + * of one worker, so the maximum is out of reach for a sound sender; + * a wrap would read as "no detached work" and hand a running worker + * back to the reaper. */ - port->detached_app = 1; - if (port->detached == 0) { - port->detached = 1; - app->detached_processes++; - changed = 1; + if (nxt_slow_path(port->detached_app == UINT32_MAX)) { + alert = "sent a detached start at the maximum count"; - /* - * The worker may already have been parked as idle. libunit - * sends the start edge before the last response message, but - * that only orders the two on the wire: the response is - * answered by the engine that owns the request, this by the - * main thread, and a request that was failed rather than - * answered -- a "limits": {"timeout"} expiry, an error -- runs - * nxt_router_app_port_release() with no start edge in sight at - * all, so the port can have been sitting in idle_ports for as - * long as the application chose to keep running. - * - * Unwind it here, exactly as an acknowledgement does. Leaving - * it in place would be worse than not having this message: the - * reaper would QUIT a worker that is still executing -- the bug - * this exists to fix -- and would clear ->app on the way, so - * nxt_router_app_port_close() would never run and the count and - * the application reference below would never be settled. - */ - - start_process = nxt_router_app_port_busy(task, app, port, - "detached"); + } else { + port->detached_app++; + + if (port->detached == 0) { + port->detached = 1; + app->detached_processes++; + changed = 1; + + /* + * The worker may already have been parked as idle. libunit + * sends the start edge before the last response message, but + * that only orders the two on the wire: the response is + * answered by the engine that owns the request, this by the + * main thread, and a request that was failed rather than + * answered -- a "limits": {"timeout"} expiry, an error -- + * runs nxt_router_app_port_release() with no start edge in + * sight at all, so the port can have been sitting in + * idle_ports for as long as the application chose to keep + * running. + * + * Unwind it here, exactly as an acknowledgement does. + * Leaving it in place would be worse than not having this + * message: the reaper would QUIT a worker that is still + * executing -- the bug this exists to fix -- and would clear + * ->app on the way, so nxt_router_app_port_close() would + * never run and the count and the application reference + * below would never be settled. + */ + + start_process = nxt_router_app_port_busy(task, app, port, + "detached"); + } } - } else { - port->detached_app = 0; + } else if (nxt_slow_path(port->detached_app == 0)) { + + /* + * An unmatched finish: a worker whose death has already settled the + * count, one otherwise out of step with the router, or a forged + * edge. Leave the count at zero and touch neither + * app->detached_processes nor the reference. + */ + + alert = "sent an unmatched detached finish"; + + } else if (--port->detached_app == 0) { /* * The port leaves the detached state only when no reason is left: @@ -8152,6 +8183,10 @@ nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) nxt_thread_mutex_unlock(&app->mutex); + if (nxt_slow_path(alert != NULL)) { + nxt_alert(task, "detached_handler: %PI %s", pid, alert); + } + if (adjust_idle_timer) { nxt_router_app_use(task, app, 1); nxt_event_engine_post(app->engine, &app->adjust_idle_work); @@ -8174,9 +8209,11 @@ nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) * nxt_request_rpc_data_unlink() has already dropped its reference, so a * configuration reload could free the application out from under a * process still executing. The finish edge that changed the state - * returns it, and an edge that changed nothing returns only its own, so - * a repeated start cannot take a second reference that the single - * finish never returns. + * returns it, and an edge that changed nothing returns only its own. + * One reference per process, then, not per counted edge: the inner + * starts and finishes take one and give it straight back, and a start + * that found the router's own mark already set leaves that reference to + * nxt_router_app_abandoned_settle(). */ drop = 1; diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 0bc59462d..7126a9e7e 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -3783,12 +3783,12 @@ nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx) lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); /* - * Close the main context: the router keeps the detached flag on the - * main port (id 0), and only closing that port settles the count - * and the application reference. This assumes one context, as in - * PHP, the only caller of nxt_unit_request_done_detached(). A - * non-main context closing alone would leave the worker flagged - * detached for the life of the process, and the main context is + * Close the main context: the router keeps the detached count on + * the main port (id 0), and only closing that port settles the + * count and the application reference. This assumes one context, + * as in PHP, the only caller of nxt_unit_request_done_detached(). + * A non-main context closing alone would leave this context's start + * counted for the life of the process, and the main context is * changed here without cross-thread synchronization. This is a * comment and not nxt_assert(): libunit does not link the thread * context that nxt_assert() needs. diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 2adb00802..9ffc10c47 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -326,10 +326,11 @@ void nxt_unit_request_done(nxt_unit_request_info_t *req, int rc); * libunit reports the work finished by itself, once the request handler * returns; the application does not have to pair this call with anything. * That covers the requests libunit hands to the handler, not one taken with - * nxt_unit_dequeue_request(). The router keeps one flag per worker, not a - * count per context, so a worker running several contexts at once has the - * first context's finish clear it. PHP's fastcgi_finish_request() is the - * caller this exists for, and PHP runs one context. + * nxt_unit_dequeue_request(). libunit tracks the state per context and the + * router counts the reports per worker, so several contexts of one worker + * may run detached work at once: the worker stays busy until the last of + * them returns. PHP's fastcgi_finish_request() is the caller this exists + * for, and PHP runs one context. */ void nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc); diff --git a/src/test/nxt_router_detached_test.c b/src/test/nxt_router_detached_test.c index 6e804af0c..5f1428e47 100644 --- a/src/test/nxt_router_detached_test.c +++ b/src/test/nxt_router_detached_test.c @@ -18,6 +18,11 @@ * only ->detached, the count and the application reference, and those are * what the test reads. Each forged edge is followed by the genuine one, which * shows that the fixture reaches the accounting at all. + * + * It also covers the counting the accounting rests on: two starts and one + * finish leave the worker detached, the second finish releases it, and a + * finish that matches no start is refused without disturbing either count. + * That last edge logs an alert, which is the expected output. */ #include @@ -30,14 +35,68 @@ #if (NXT_USE_CMSG_PID) +/* Which credential the edge arrives with. */ + +#define NXT_ROUTER_DETACHED_TEST_SELF 0 +#define NXT_ROUTER_DETACHED_TEST_OTHER 1 +#define NXT_ROUTER_DETACHED_TEST_NONE 2 + + +typedef struct { + uint8_t state; + uint8_t sender; + uint32_t detached_app; + uint8_t detached; + uint32_t processes; + nxt_atomic_t use_count; + const char *name; +} nxt_router_detached_test_edge_t; + + +/* + * The edges, and the accounting each one must leave behind. The start and + * the finish are a counted pair: only the first start and the last finish + * move ->detached, app->detached_processes and the application reference, + * and a finish that matches no start moves none of them. + */ + +static const nxt_router_detached_test_edge_t + nxt_router_detached_test_edges[] = +{ + { NXT_PORT_DETACHED_START, NXT_ROUTER_DETACHED_TEST_OTHER, 0, 0, 0, 8, + "a forged start" }, + { NXT_PORT_DETACHED_START, NXT_ROUTER_DETACHED_TEST_SELF, 1, 1, 1, 9, + "a genuine start" }, + { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_OTHER, 1, 1, 1, 9, + "a forged finish" }, + { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_NONE, 1, 1, 1, 9, + "a finish with no credential" }, + { NXT_PORT_DETACHED_START, NXT_ROUTER_DETACHED_TEST_SELF, 2, 1, 1, 9, + "a second start" }, + { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_SELF, 1, 1, 1, 9, + "the first of two finishes" }, + { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_SELF, 0, 0, 0, 8, + "the last of two finishes" }, + { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_SELF, 0, 0, 0, 8, + "an unmatched finish" }, + { NXT_PORT_DETACHED_START, NXT_ROUTER_DETACHED_TEST_SELF, 1, 1, 1, 9, + "a start after an unmatched finish" }, + { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_SELF, 0, 0, 0, 8, + "the matching finish" }, +}; + + static nxt_int_t nxt_router_detached_test_edge(nxt_thread_t *thr, nxt_task_t *task, - nxt_port_t *port, nxt_app_t *app, uint8_t state, nxt_pid_t sender, - nxt_bool_t detached, nxt_atomic_t use_count, const char *name) + nxt_port_t *port, nxt_app_t *app, + const nxt_router_detached_test_edge_t *e, nxt_pid_t sender) { + uint8_t state; nxt_buf_t b; nxt_port_recv_msg_t msg; + state = e->state; + nxt_memzero(&b, sizeof(nxt_buf_t)); b.mem.start = &state; b.mem.pos = &state; @@ -56,16 +115,19 @@ nxt_router_detached_test_edge(nxt_thread_t *thr, nxt_task_t *task, nxt_router_test_detached_handler(task, &msg); - if (port->detached != detached - || app->detached_processes != (detached ? 1 : 0) - || app->use_count != use_count) + if (port->detached_app != e->detached_app + || port->detached != e->detached + || app->detached_processes != e->processes + || app->use_count != e->use_count) { nxt_log_error(NXT_LOG_NOTICE, thr->log, - "router detached test: %s: detached %d, count %uD, " - "use count %d; expected %d, %d, %d", - name, (int) port->detached, app->detached_processes, - (int) app->use_count, (int) detached, - detached ? 1 : 0, (int) use_count); + "router detached test: %s: detached_app %uD, " + "detached %d, count %uD, use count %d; " + "expected %uD, %d, %uD, %d", + e->name, port->detached_app, (int) port->detached, + app->detached_processes, (int) app->use_count, + e->detached_app, (int) e->detached, e->processes, + (int) e->use_count); return NXT_ERROR; } @@ -90,13 +152,16 @@ nxt_router_detached_test(nxt_thread_t *thr) nxt_mp_t *mp; nxt_app_t *app; nxt_int_t ret; - nxt_pid_t pid, other; + nxt_uint_t i; + nxt_pid_t pid, other, sender; nxt_task_t *task; nxt_port_t *port; nxt_bool_t app_mutex, hashed; nxt_runtime_t *rt, *saved_rt; nxt_event_engine_t engine, *saved_engine; + const nxt_router_detached_test_edge_t *e; + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "router detached test started"); @@ -169,28 +234,29 @@ nxt_router_detached_test(nxt_thread_t *thr) hashed = 1; - if (nxt_router_detached_test_edge(thr, task, port, app, - NXT_PORT_DETACHED_START, other, 0, 8, - "a forged start") - != NXT_OK - || nxt_router_detached_test_edge(thr, task, port, app, - NXT_PORT_DETACHED_START, pid, 1, 9, - "a genuine start") - != NXT_OK - || nxt_router_detached_test_edge(thr, task, port, app, - NXT_PORT_DETACHED_FINISH, other, 1, 9, - "a forged finish") - != NXT_OK - || nxt_router_detached_test_edge(thr, task, port, app, - NXT_PORT_DETACHED_FINISH, -1, 1, 9, - "a finish with no credential") - != NXT_OK - || nxt_router_detached_test_edge(thr, task, port, app, - NXT_PORT_DETACHED_FINISH, pid, 0, 8, - "a genuine finish") - != NXT_OK) - { - goto done; + for (i = 0; i < nxt_nitems(nxt_router_detached_test_edges); i++) { + e = &nxt_router_detached_test_edges[i]; + + switch (e->sender) { + case NXT_ROUTER_DETACHED_TEST_SELF: + sender = pid; + break; + + case NXT_ROUTER_DETACHED_TEST_OTHER: + sender = other; + break; + + default: + sender = -1; + break; + } + + if (nxt_slow_path(nxt_router_detached_test_edge(thr, task, port, app, + e, sender) + != NXT_OK)) + { + goto done; + } } nxt_log_error(NXT_LOG_NOTICE, thr->log, "router detached test passed"); From 642c9c8e11918d3cc909bcf9f7ceedc1f3f5ef87 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 22:44:19 +0000 Subject: [PATCH 277/391] router: log a refused detached edge as an alert once per port An unmatched FINISH, or a START at the maximum count, is a valid message that an application can send in a loop, and each one logged an alert. Add a one-byte flag to the port so that only the first refused edge on a port is an alert; the rest go to the debug log. The flag sits in the padding after ->detached, so the port does not grow. Also say in comments why only detached_app saturates, and why the FINISH retry in libunit is safe only while a failed send delivers nothing. The router C test gets per-edge presets and covers the two branches the reference accounting turns on: a START while the router holds its own mark, and a START at UINT32_MAX. The expectation table uses nxt_atomic_uint_t instead of the volatile nxt_atomic_t. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- src/nxt_port.h | 14 ++- src/nxt_router.c | 15 ++- src/nxt_unit.c | 12 +++ src/test/nxt_router_detached_test.c | 137 +++++++++++++++++++++------- 4 files changed, 142 insertions(+), 36 deletions(-) diff --git a/src/nxt_port.h b/src/nxt_port.h index 72d94e3a3..71facf487 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -398,6 +398,15 @@ struct nxt_port_s { */ uint8_t detached; + /* + * An edge the accounting refused -- an unmatched FINISH, or a START at + * the maximum count -- was already logged as an alert for this port. + * Such an edge is a valid message the application can send in a loop, + * so only the first one is an alert and the rest go to the debug log. + * Touched only by nxt_router_detached_apply(), on the main thread. + */ + uint8_t detached_alerted; + /* * How many units of detached work the application reported of its own: * one for each START edge whose FINISH edge has not arrived. Until the @@ -405,7 +414,10 @@ struct nxt_port_s { * request left. A count rather than a flag because the edges carry no * context id and a worker may run several contexts -- see * nxt_port_detached_t above -- so the first context's FINISH must not - * speak for the rest. + * speak for the rest. It saturates at UINT32_MAX and detached_router + * below does not: this one moves on edges the application sends, which + * the router cannot bound, while detached_router moves only on requests + * the router itself gave up on, one per request it has in flight. */ uint32_t detached_app; diff --git a/src/nxt_router.c b/src/nxt_router.c index c948e13b2..d99010e97 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -8035,10 +8035,10 @@ static void nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) { int drop; - const char *alert; nxt_app_t *app; nxt_port_t *port; nxt_bool_t changed, start_process, adjust_idle_timer; + const char *alert; nxt_runtime_t *rt; nxt_atomic_int_t c; @@ -8183,8 +8183,19 @@ nxt_router_detached_apply(nxt_task_t *task, nxt_pid_t pid, uint8_t state) nxt_thread_mutex_unlock(&app->mutex); + /* + * The application can send a refused edge in a loop, so it is an alert + * once per port and debug output after that. + */ + if (nxt_slow_path(alert != NULL)) { - nxt_alert(task, "detached_handler: %PI %s", pid, alert); + if (port->detached_alerted == 0) { + port->detached_alerted = 1; + nxt_alert(task, "detached_handler: %PI %s", pid, alert); + + } else { + nxt_debug(task, "detached_handler: %PI %s", pid, alert); + } } if (adjust_idle_timer) { diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 7126a9e7e..077b5ab79 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -3767,6 +3767,18 @@ nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx) return NXT_UNIT_OK; } + /* + * A resend is safe only because a failed send delivered nothing. The + * router counts the edges and they name no context, so a FINISH that + * arrived twice would settle another context's START and hand a worker + * that is still executing back to the idle economy. + * nxt_unit_send_detached() fails on a missing router port, on a queue + * overflow, which enqueues nothing, and on a short send, which a port + * socket (SOCK_DGRAM or SOCK_SEQPACKET) makes all or nothing; none of + * these delivers the edge. Keep it that way, or give the edge a + * context id first. + */ + res = nxt_unit_send_detached(ctx, NXT_PORT_DETACHED_FINISH); if (nxt_fast_path(res == NXT_UNIT_OK)) { ctx_impl->detached = NXT_UNIT_DETACHED_NONE; diff --git a/src/test/nxt_router_detached_test.c b/src/test/nxt_router_detached_test.c index 5f1428e47..1153bd0be 100644 --- a/src/test/nxt_router_detached_test.c +++ b/src/test/nxt_router_detached_test.c @@ -22,7 +22,10 @@ * It also covers the counting the accounting rests on: two starts and one * finish leave the worker detached, the second finish releases it, and a * finish that matches no start is refused without disturbing either count. - * That last edge logs an alert, which is the expected output. + * The first such edge logs an alert, which is the expected output; later ones + * go to the debug log. Two presets reach the branches the application's + * edges alone do not: a start while the router holds its own mark, and a + * start at the maximum count. */ #include @@ -42,47 +45,105 @@ #define NXT_ROUTER_DETACHED_TEST_NONE 2 +/* + * The accounting an edge finds. An edge with a preset starts from it rather + * than from what the previous edge left, which is how the test reaches the + * states only the router's own path or a long run of edges would produce. + */ + +typedef struct { + nxt_atomic_uint_t use_count; + uint32_t detached_app; + uint32_t detached_router; + uint32_t processes; + uint8_t detached; +} nxt_router_detached_test_state_t; + + typedef struct { - uint8_t state; - uint8_t sender; - uint32_t detached_app; - uint8_t detached; - uint32_t processes; - nxt_atomic_t use_count; - const char *name; + const nxt_router_detached_test_state_t *preset; + const char *name; + uint32_t detached_app; + uint32_t processes; + nxt_atomic_uint_t use_count; + uint8_t state; + uint8_t sender; + uint8_t detached; + uint8_t alerted; } nxt_router_detached_test_edge_t; +/* + * What nxt_router_app_abandon() leaves on an idle port: the router's own + * mark, the detached state it set, and the application reference that state + * holds. + */ + +static const nxt_router_detached_test_state_t + nxt_router_detached_test_router_mark = { 9, 0, 1, 1, 1 }; + + +/* A worker whose count has reached the maximum. */ + +static const nxt_router_detached_test_state_t + nxt_router_detached_test_saturated = { 9, UINT32_MAX, 0, 1, 1 }; + + +#define NXT_RDT_START NXT_PORT_DETACHED_START +#define NXT_RDT_FINISH NXT_PORT_DETACHED_FINISH +#define NXT_RDT_SELF NXT_ROUTER_DETACHED_TEST_SELF +#define NXT_RDT_OTHER NXT_ROUTER_DETACHED_TEST_OTHER +#define NXT_RDT_NONE NXT_ROUTER_DETACHED_TEST_NONE + + /* * The edges, and the accounting each one must leave behind. The start and * the finish are a counted pair: only the first start and the last finish * move ->detached, app->detached_processes and the application reference, - * and a finish that matches no start moves none of them. + * and a finish that matches no start moves none of them. A refused edge is + * an alert the first time on a port only. + * + * A start that finds the router's own mark already set changes no state, so + * the reference it took goes straight back: the state's reference belongs to + * nxt_router_app_abandoned_settle(). The finish that follows leaves the + * state to the router too. A start at the maximum count is refused rather + * than wrapped to "no detached work". */ static const nxt_router_detached_test_edge_t nxt_router_detached_test_edges[] = { - { NXT_PORT_DETACHED_START, NXT_ROUTER_DETACHED_TEST_OTHER, 0, 0, 0, 8, - "a forged start" }, - { NXT_PORT_DETACHED_START, NXT_ROUTER_DETACHED_TEST_SELF, 1, 1, 1, 9, - "a genuine start" }, - { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_OTHER, 1, 1, 1, 9, - "a forged finish" }, - { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_NONE, 1, 1, 1, 9, - "a finish with no credential" }, - { NXT_PORT_DETACHED_START, NXT_ROUTER_DETACHED_TEST_SELF, 2, 1, 1, 9, - "a second start" }, - { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_SELF, 1, 1, 1, 9, - "the first of two finishes" }, - { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_SELF, 0, 0, 0, 8, - "the last of two finishes" }, - { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_SELF, 0, 0, 0, 8, - "an unmatched finish" }, - { NXT_PORT_DETACHED_START, NXT_ROUTER_DETACHED_TEST_SELF, 1, 1, 1, 9, - "a start after an unmatched finish" }, - { NXT_PORT_DETACHED_FINISH, NXT_ROUTER_DETACHED_TEST_SELF, 0, 0, 0, 8, - "the matching finish" }, + { NULL, "a forged start", + 0, 0, 8, NXT_RDT_START, NXT_RDT_OTHER, 0, 0 }, + { NULL, "a genuine start", + 1, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 0 }, + { NULL, "a forged finish", + 1, 1, 9, NXT_RDT_FINISH, NXT_RDT_OTHER, 1, 0 }, + { NULL, "a finish with no credential", + 1, 1, 9, NXT_RDT_FINISH, NXT_RDT_NONE, 1, 0 }, + { NULL, "a second start", + 2, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 0 }, + { NULL, "the first of two finishes", + 1, 1, 9, NXT_RDT_FINISH, NXT_RDT_SELF, 1, 0 }, + { NULL, "the last of two finishes", + 0, 0, 8, NXT_RDT_FINISH, NXT_RDT_SELF, 0, 0 }, + { NULL, "an unmatched finish", + 0, 0, 8, NXT_RDT_FINISH, NXT_RDT_SELF, 0, 1 }, + { NULL, "a second unmatched finish", + 0, 0, 8, NXT_RDT_FINISH, NXT_RDT_SELF, 0, 1 }, + { NULL, "a start after an unmatched finish", + 1, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 1 }, + { NULL, "the matching finish", + 0, 0, 8, NXT_RDT_FINISH, NXT_RDT_SELF, 0, 1 }, + { &nxt_router_detached_test_router_mark, + "a start while the router holds its own mark", + 1, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 1 }, + { NULL, "a finish while the router holds its own mark", + 0, 1, 9, NXT_RDT_FINISH, NXT_RDT_SELF, 1, 1 }, + { &nxt_router_detached_test_saturated, "a start at the maximum count", + UINT32_MAX, 1, 9, NXT_RDT_START, NXT_RDT_SELF, 1, 1 }, + { NULL, "a finish below the maximum count", + UINT32_MAX - 1, 1, 9, NXT_RDT_FINISH, NXT_RDT_SELF, 1, 1 }, }; @@ -113,20 +174,30 @@ nxt_router_detached_test_edge(nxt_thread_t *thr, nxt_task_t *task, msg.port_msg.last = 1; msg.cmsg_pid = sender; + if (e->preset != NULL) { + port->detached_app = e->preset->detached_app; + port->detached_router = e->preset->detached_router; + port->detached = e->preset->detached; + app->detached_processes = e->preset->processes; + app->use_count = e->preset->use_count; + } + nxt_router_test_detached_handler(task, &msg); if (port->detached_app != e->detached_app || port->detached != e->detached + || port->detached_alerted != e->alerted || app->detached_processes != e->processes || app->use_count != e->use_count) { nxt_log_error(NXT_LOG_NOTICE, thr->log, "router detached test: %s: detached_app %uD, " - "detached %d, count %uD, use count %d; " - "expected %uD, %d, %uD, %d", + "detached %d, alerted %d, count %uD, use count %d; " + "expected %uD, %d, %d, %uD, %d", e->name, port->detached_app, (int) port->detached, - app->detached_processes, (int) app->use_count, - e->detached_app, (int) e->detached, e->processes, + (int) port->detached_alerted, app->detached_processes, + (int) app->use_count, e->detached_app, + (int) e->detached, (int) e->alerted, e->processes, (int) e->use_count); return NXT_ERROR; } From 4f8cbe591e00685731de4c10557d9cf07070bae3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:34:25 +0000 Subject: [PATCH 278/391] otel: pass FreeUnit's own span to the application as its parent With telemetry on, the application got the client's traceparent unchanged, so its spans were siblings of FreeUnit's span and not its children. Replace every inbound traceparent (valid or not) with one that carries FreeUnit's trace id and its own span id as parent-id. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- src/nxt_otel.c | 56 ++++++++++------- test/php/traceparent/index.php | 3 + test/python/traceparent/wsgi.py | 7 +++ test/test_otel.py | 21 ++++++- test/test_otel_traceparent_app.py | 89 +++++++++++++++++++++++++++ test/test_otel_traceparent_app_php.py | 29 +++++++++ 6 files changed, 180 insertions(+), 25 deletions(-) create mode 100644 test/php/traceparent/index.php create mode 100644 test/python/traceparent/wsgi.py create mode 100644 test/test_otel_traceparent_app.py create mode 100644 test/test_otel_traceparent_app_php.py diff --git a/src/nxt_otel.c b/src/nxt_otel.c index 99a11cbcc..2d9c33c9c 100644 --- a/src/nxt_otel.c +++ b/src/nxt_otel.c @@ -186,36 +186,46 @@ nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r) return; } - if (r->otel->trace_id != NULL) { - // copy in the pre-existing traceparent for the response - sprintf((char *) traceval, "%s-%s-%s-%s", - (char *) r->otel->version, - (char *) r->otel->trace_id, - (char *) r->otel->parent_id, - (char *) r->otel->trace_flags); - /* - * if we didn't inherit a trace id then we need to add the - * traceparent header to the request + * The traceparent forwarded to the peer/app -- and echoed in the + * response -- always carries this request's own span as parent-id, so + * that the app's spans are children of ours rather than siblings of the + * inbound parent. nxt_otel_rs_copy_traceparent() reads trace-id, + * span-id and flags off the span's own context, which already carries + * the inherited trace-id/flags when a valid inbound traceparent was + * accepted, or freshly generated ones otherwise -- so one call covers + * both cases. */ - } else { - - nxt_otel_rs_copy_traceparent(traceval, r->otel->trace); + nxt_otel_rs_copy_traceparent(traceval, r->otel->trace); - /* - * nxt_list_add() hands out non-zeroed memory: garbage skip/hopbyhop - * bits make the peer/app serializers randomly drop the field. - */ - f = nxt_http_req_field_zero_add(r); - if (nxt_slow_path(f == NULL)) { - return; + /* + * Any inbound traceparent, valid or not, is replaced by the value + * below, so the peer/app never sees two of them. + */ + nxt_http_fields_each(f, r->inline_fields, r->num_inline_fields, r->fields) + { + if (f->name_length == nxt_length("traceparent") + && nxt_memcasecmp(f->name, "traceparent", + nxt_length("traceparent")) == 0) + { + f->skip = 1; } - nxt_http_field_name_set(f, "traceparent"); - f->value = traceval; - f->value_length = nxt_strlen(traceval); + } nxt_http_fields_loop; + + /* + * nxt_list_add() hands out non-zeroed memory: garbage skip/hopbyhop + * bits make the peer/app serializers randomly drop the field. + */ + f = nxt_http_req_field_zero_add(r); + if (nxt_slow_path(f == NULL)) { + return; } + nxt_http_field_name_set(f, "traceparent"); + f->value = traceval; + f->value_length = nxt_strlen(traceval); + f = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); if (nxt_slow_path(f == NULL)) { nxt_log(task, NXT_LOG_ERR, diff --git a/test/php/traceparent/index.php b/test/php/traceparent/index.php new file mode 100644 index 000000000..59c893eb0 --- /dev/null +++ b/test/php/traceparent/index.php @@ -0,0 +1,3 @@ + Date: Sat, 26 Sep 2026 11:31:22 +0000 Subject: [PATCH 279/391] test: fix tests and fixtures that could not fail - test_listeners_unix_abstract: drop two asserts that sat after an unconditional pytest.skip('not yet') and never ran. - unit_stop(): without --restart it now only returns. The hidden pytest.skip('no restart mode') is gone; a new requires_restart fixture skips such tests at setup. test_asgi_lifespan, test_asgi_lifespan_targets and test_state_store_full_filesystem use it. The inspect import is no longer needed. - Remove option.skip_sanitizer. Nothing set it, so the sanitizer check in Log.check_alerts() now runs always. No behaviour change. - test_python_process_switch and test_asgi_process_switch: assert the status of the last request instead of ignoring it. - nxt_malloc_test: log "malloc test passed" like every other C test. - test_asgi_lifespan: create the cookie files with touch() and chmod(0o777). touch(0o777) applies the umask, so the application running as "nobody" could not write "version" and the test failed with --restart. - CI: run test_asgi_lifespan.py with --restart on the python legs. No CI run passed --restart to this file before, so both lifespan tests were always skipped. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 10 ++++++++++ src/test/nxt_malloc_test.c | 2 ++ test/conftest.py | 14 +++++++++----- test/test_asgi_application.py | 2 +- test/test_asgi_lifespan.py | 9 ++++++--- test/test_configuration.py | 9 +++------ test/test_python_application.py | 2 +- test/test_state_store.py | 2 +- test/unit/log.py | 5 ++--- test/unit/option.py | 1 - 10 files changed, 35 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b24582f2e..0ec743280 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -821,6 +821,16 @@ jobs: /usr/bin/pytest-3 --print-log test/test_capget_fallback.py if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} + # test_asgi_lifespan.py stops unitd from inside the test to check the + # shutdown event. That needs --restart, which the run above does not + # pass, so both lifespan tests skip there. Run the file again with it. + - name: Run ASGI lifespan tests with --restart + run: | + export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" + sudo -E /usr/bin/pytest-3 --print-log --restart \ + test/test_asgi_lifespan.py + if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} + # C unit-test suite (src/test/ -> build/tests): HTTP parser, port/IPC # fault-injection, lvlhsh, mp, rbtree, base64, utf8, clone creds, etc. # Previously compiled but never executed in CI; runs here on every PR as its diff --git a/src/test/nxt_malloc_test.c b/src/test/nxt_malloc_test.c index daa609b14..e7c34cdf3 100644 --- a/src/test/nxt_malloc_test.c +++ b/src/test/nxt_malloc_test.c @@ -119,5 +119,7 @@ nxt_malloc_test(nxt_thread_t *thr) s[-1].size + 1, s->size, s->alignment, s->tight); } + nxt_log_error(NXT_LOG_NOTICE, thr->log, "malloc test passed"); + return NXT_OK; } diff --git a/test/conftest.py b/test/conftest.py index 2ece98916..d685610d2 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,5 @@ import atexit import fcntl -import inspect import json import os import re @@ -216,7 +215,6 @@ def run(request): r'sendmsg.+failed', r'recvmsg.+failed', ] - option.skip_sanitizer = False _fds_info['main']['skip'] = False _fds_info['router']['skip'] = False @@ -600,9 +598,6 @@ def unit_run(state_dir=None): def unit_stop(): if not option.restart: - if inspect.stack()[1].function.startswith('test_'): - pytest.skip('no restart mode') - return # Startup may have failed before the process/pid were recorded; nothing to @@ -978,6 +973,15 @@ def is_unsafe(request): return request.config.getoption("--unsafe") +@pytest.fixture +def requires_restart(): + # unit_stop() does nothing without --restart, so a test that calls it in + # its body cannot check anything after the call. Skip such a test at + # setup. + if not option.restart: + pytest.skip('no restart mode') + + @pytest.fixture def require(): return check_prerequisites diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 226a1ed75..c0888f265 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -394,7 +394,7 @@ def test_asgi_process_switch(): for _ in range(10): client.get(headers=headers_delay_1, no_recv=True) - client.get(headers=headers_delay_1) + assert client.get(headers=headers_delay_1)['status'] == 200 def test_asgi_application_loading_error(skip_alert): diff --git a/test/test_asgi_lifespan.py b/test/test_asgi_lifespan.py index e09ea1cc4..9f4409ef7 100644 --- a/test/test_asgi_lifespan.py +++ b/test/test_asgi_lifespan.py @@ -34,10 +34,13 @@ def setup_cookies(prefix): for name in ['startup', 'shutdown', 'version']: path = Path(f'{option.test_dir}/python/lifespan/empty/{prefix}{name}') - path.touch(0o777) + path.touch() + # touch() applies the umask; the application runs as "nobody" and + # must be able to write "version". + path.chmod(0o777) -def test_asgi_lifespan(): +def test_asgi_lifespan(requires_restart): client.load('lifespan/empty') setup_cookies('') @@ -49,7 +52,7 @@ def test_asgi_lifespan(): assert_cookies('') -def test_asgi_lifespan_targets(): +def test_asgi_lifespan_targets(requires_restart): path = f'{option.test_dir}/python/lifespan/empty' assert 'success' in client.conf( diff --git a/test/test_configuration.py b/test/test_configuration.py index e490b32d9..d4a7ed59c 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -365,13 +365,10 @@ def test_listeners_no_app(): def test_listeners_unix_abstract(system): - if system != 'Linux': - assert 'error' in try_addr("unix:@sock"), 'abstract at' - - pytest.skip('not yet') + if system == 'Linux': + pytest.skip('not yet') - assert 'error' in try_addr("unix:\0soc"), 'abstract \0' - assert 'error' in try_addr("unix:\u0000soc"), 'abstract \0 unicode' + assert 'error' in try_addr("unix:@sock"), 'abstract at' def test_listeners_addr(): diff --git a/test/test_python_application.py b/test/test_python_application.py index 3d49d8772..b87a914dd 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -387,7 +387,7 @@ def test_python_process_switch(): for _ in range(10): client.get(headers=headers_delay_1, no_recv=True) - client.get(headers=headers_delay_1) + assert client.get(headers=headers_delay_1)['status'] == 200 @pytest.mark.skip('not yet') diff --git a/test/test_state_store.py b/test/test_state_store.py index 75f015be3..1fa771903 100644 --- a/test/test_state_store.py +++ b/test/test_state_store.py @@ -66,7 +66,7 @@ def wait_for_stored(statedir, expected, wait=100): return None -def test_state_store_full_filesystem(skip_alert): +def test_state_store_full_filesystem(requires_restart, skip_alert): """A store that runs out of space must not damage the stored config.""" if os.geteuid() != 0: diff --git a/test/unit/log.py b/test/unit/log.py index 7d7e355aa..c5812f0f9 100644 --- a/test/unit/log.py +++ b/test/unit/log.py @@ -44,10 +44,9 @@ def check_alerts(log=None): assert not alerts, 'alert(s)' - if not option.skip_sanitizer: - sanitizer_errors = re.findall('.+Sanitizer.+', log) + sanitizer_errors = re.findall('.+Sanitizer.+', log) - assert not sanitizer_errors, 'sanitizer error(s)' + assert not sanitizer_errors, 'sanitizer error(s)' if found and option.detailed: print('skipped.') diff --git a/test/unit/option.py b/test/unit/option.py index 7c66c6193..40f2057e8 100644 --- a/test/unit/option.py +++ b/test/unit/option.py @@ -9,7 +9,6 @@ class Options: 'configure_flag': {}, 'is_privileged': os.geteuid() == 0, 'skip_alerts': [], - 'skip_sanitizer': False, 'system': platform.system(), } From 81a7106a35059cab4a590f9b06f9d67d395bc301 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 11:57:17 +0000 Subject: [PATCH 280/391] ci: run the unsafe websocket cases and the unprivileged configuration tests No CI job passed --unsafe, so the long Autobahn 9.x websocket cases never ran. test_unprivileged_user_error needs a unitd that is not root, so it never ran either. - Replace the "is_unsafe" fixture and the in-body skips with one skipif on option.unsafe. Give option.unsafe a default. - Pass --unsafe in every language leg. - Run test_configuration.py in the unprivileged python step. - test_applications_type_only: remove the skip, it passes now. - test_listeners_empty: skip becomes a non-strict xfail. Validation still accepts an empty listener; the router then fails to apply it. The ASGI and java 9.x cases pass now that #429 and #430 are fixed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 18 +++++++++++------- test/conftest.py | 5 ----- test/test_asgi_websockets.py | 7 +++---- test/test_configuration.py | 6 ++++-- test/test_java_websockets.py | 7 +++---- test/test_node_websockets.py | 7 +++---- test/unit/option.py | 1 + 7 files changed, 25 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 0ec743280..ad7c59841 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -779,11 +779,12 @@ jobs: # The wasm legs build their guest fixture as the runner user, so # they run unprivileged to keep the result reachable by pytest; the # others install applications as root. + # --unsafe runs the long Autobahn 9.x websocket cases. if [ "${{ matrix.build }}" == "wasm-wasi-component" ] || [ "${{ matrix.build }}" == "wasm" ]; then set -o pipefail - pytest-3 --print-log ${{ steps.metadata.outputs.testpath }} 2>&1 | tee "$RUNNER_TEMP/pytest.out" + pytest-3 --print-log --unsafe ${{ steps.metadata.outputs.testpath }} 2>&1 | tee "$RUNNER_TEMP/pytest.out" else - sudo -E pytest-3 --print-log ${{ steps.metadata.outputs.testpath }} + sudo -E pytest-3 --print-log --unsafe ${{ steps.metadata.outputs.testpath }} fi # check_prerequisites() skips the whole file when the clang/wasi-sysroot @@ -806,19 +807,22 @@ jobs: # real application to show it inherits the shim harmlessly, and this # leg is where a language module exists. # + # test_configuration.py runs here too: test_unprivileged_user_error + # needs a unitd that is not root. + # # pytest-3 is named absolutely: setup-python puts a toolcache bin # directory ahead of /usr/bin for this step and not for the sudo'd one # above, and both must run the pytest that "apt install python3-pytest" # provided. # # !cancelled() rather than the implicit success(): this is the only leg - # that ever runs the file, and gating it on the whole sudo'd suite above - # would let one unrelated flake silently take that coverage away again -- - # the very problem this step exists to fix. - - name: Run capget fallback tests unprivileged + # that ever runs these files, and gating it on the whole sudo'd suite + # above would let one unrelated flake silently take that coverage away + # again -- the very problem this step exists to fix. + - name: Run capget fallback and configuration tests unprivileged run: | export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" - /usr/bin/pytest-3 --print-log test/test_capget_fallback.py + /usr/bin/pytest-3 --print-log test/test_capget_fallback.py test/test_configuration.py if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} # test_asgi_lifespan.py stops unitd from inside the test to check the diff --git a/test/conftest.py b/test/conftest.py index d685610d2..b2ca6bee5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -968,11 +968,6 @@ def is_su(): return option.is_privileged -@pytest.fixture -def is_unsafe(request): - return request.config.getoption("--unsafe") - - @pytest.fixture def requires_restart(): # unit_stop() does nothing without --restart, so a test that calls it in diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index 65ee5153d..b9c24bc3a 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -6,6 +6,7 @@ from unit.applications.lang.python import ApplicationPython from unit.applications.websockets import ApplicationWebsocket +from unit.option import option prerequisites = { 'modules': {'python': lambda v: version.parse(v) >= version.parse('3.5')} @@ -1343,10 +1344,8 @@ def test_asgi_websockets_7_13_1__7_13_2(): check_close(sock, 1002) -def test_asgi_websockets_9_1_1__9_6_6(is_unsafe, system): - if not is_unsafe: - pytest.skip('unsafe, long run') - +@pytest.mark.skipif(not option.unsafe, reason='unsafe, long run') +def test_asgi_websockets_9_1_1__9_6_6(system): client.load('websockets/mirror') assert 'success' in client.conf( diff --git a/test/test_configuration.py b/test/test_configuration.py index d4a7ed59c..b48ef9bb8 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -219,7 +219,6 @@ def test_applications_string(): assert 'error' in client.conf('"{}"', 'applications'), 'string' -@pytest.mark.skip('not yet, unsafe') def test_applications_type_only(): assert 'error' in client.conf( {"app": {"type": "python"}}, 'applications' @@ -353,7 +352,10 @@ def conf_log(value): ), 'path valid' -@pytest.mark.skip('not yet, unsafe') +@pytest.mark.xfail( + reason='validation accepts an empty listener; the router then fails', + strict=False, +) def test_listeners_empty(): assert 'error' in client.conf({"*:8080": {}}, 'listeners'), 'listener empty' diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 50b55f802..a25731eac 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -5,6 +5,7 @@ from unit.applications.lang.java import ApplicationJava from unit.applications.websockets import ApplicationWebsocket +from unit.option import option prerequisites = {'modules': {'java': 'any'}} @@ -1264,10 +1265,8 @@ def test_java_websockets_7_13_1__7_13_2(): check_close(sock, 1002) -def test_java_websockets_9_1_1__9_6_6(is_unsafe, system): - if not is_unsafe: - pytest.skip('unsafe, long run') - +@pytest.mark.skipif(not option.unsafe, reason='unsafe, long run') +def test_java_websockets_9_1_1__9_6_6(system): client.load('websockets_mirror') assert 'success' in client.conf( diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index dde3452af..ccfa18748 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -6,6 +6,7 @@ from unit.applications.lang.node import ApplicationNode from unit.applications.websockets import ApplicationWebsocket from unit.control import Control +from unit.option import option prerequisites = {'modules': {'node': 'any'}} @@ -1263,10 +1264,8 @@ def test_node_websockets_7_13_1__7_13_2(): check_close(sock, 1002) -def test_node_websockets_9_1_1__9_6_6(is_unsafe, system): - if not is_unsafe: - pytest.skip('unsafe, long run') - +@pytest.mark.skipif(not option.unsafe, reason='unsafe, long run') +def test_node_websockets_9_1_1__9_6_6(system): client.load('websockets/mirror') assert 'success' in client.conf( diff --git a/test/unit/option.py b/test/unit/option.py index 40f2057e8..ab9ff65eb 100644 --- a/test/unit/option.py +++ b/test/unit/option.py @@ -10,6 +10,7 @@ class Options: 'is_privileged': os.geteuid() == 0, 'skip_alerts': [], 'system': platform.system(), + 'unsafe': False, } def __setattr__(self, name, value): From cefaf8b4971095ca4b00e1e3f569457ed1976801 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 12:46:20 +0000 Subject: [PATCH 281/391] test(java): gate the async 16 MiB case on option.unsafe This branch drops the is_unsafe fixture in favour of option.unsafe, but test_java_websockets_async_16m reached master after the branch was cut and still asked for the fixture, so every Java leg failed at its setup with "fixture 'is_unsafe' not found". Skip it the same way as the other unsafe cases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YSMMShDBn9zs2gFfTrLqwA --- test/test_java_websockets.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index a25731eac..97b966180 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -1694,10 +1694,8 @@ def test_java_websockets_async_threads(): close_connection(sock) -def test_java_websockets_async_16m(is_unsafe): - if not is_unsafe: - pytest.skip('unsafe, long run') - +@pytest.mark.skipif(not option.unsafe, reason='unsafe, long run') +def test_java_websockets_async_16m(): # 16 MiB is exactly the default send cap, so it is one frame. Above the # cap a message is sent 8 KiB at a time, each part started from the # previous one's completion; lowering the cap to 8 KiB makes 2048 parts, From 9829884cf706ce8bc157fb702431875c72cca0df Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 16:06:39 +0200 Subject: [PATCH 282/391] port: pace an ENOMEM retry instead of re-checking it on the spot ENOBUFS and ENOMEM leave the socket writable, so no edge is coming and the retry has to force the readiness re-check itself. Forced on the spot it comes straight back, and while the shortage lasts the engine thread runs a tight epoll_wait/sendmsg/epoll_ctl loop. The re-arm now goes through a timer on port->engine. The delay doubles from 1ms to a 32ms cap, and a send that succeeds resets it. The message is still never dropped; the retry is only no longer made as fast as the kernel can refuse it. The owed marker is paced the same way, in nxt_port_rearm_now(), the only place it is attempted. nxt_port_announce() clears port->announce only on a send that fully succeeded, so a marker that kept failing spun on its own, whether or not the queued path was paced. The marker is attempted before the write event is enabled, so a paced one does not cost an enable and a disable in the same pass. While the timer is armed nxt_port_rearm_now() does nothing: the timer already owes the port both the enable and the marker. The armed state is retry_timer.enabled. nxt_timer_add() sets it, nxt_timer_expire() leaves it set while the timer waits in the work queue, and nxt_timer_handler() clears it on the line before it calls the handler. An armed timer holds a port reference, and nothing cancels it. A cancel cannot come from another engine, and one made here would hand nxt_timer_handler() a pointer into a pool the reference drop had just released. The timer fires within the cap, finds a closed port by its pair[1], does nothing with it, and drops the reference. An engine freed with the timer in its tree leaks the port; that is the one case left. New legs in src/test/nxt_port_fail_test.c hold an ENOMEM on the real epoll-edge fixture and count write-handler dispatches over a 64ms window, with the clock driven by the test rather than read from the machine: leg 6, queued marker 1088 before, 7 after leg 7, owed marker 1088 before, 0 after leg 8, re-armed every millisecond 1088 before, 0 after Leg 7 also checks the delay is reset once the marker goes out. Fixes: https://github.com/freeunitorg/freeunit/issues/407 Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- src/nxt_port.h | 33 ++++ src/nxt_port_socket.c | 232 +++++++++++++++++++++--- src/test/nxt_port_fail_test.c | 322 +++++++++++++++++++++++++++++++++- 3 files changed, 564 insertions(+), 23 deletions(-) diff --git a/src/nxt_port.h b/src/nxt_port.h index 71facf487..630102a31 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -477,6 +477,30 @@ struct nxt_port_s { nxt_atomic_t rearm_pending; nxt_atomic_t announce; + /* + * The pacing of a re-arm that follows a send which failed for want of + * kernel memory. + * + * ENOBUFS and ENOMEM leave the socket writable, so no edge is coming and + * the retry has to force the readiness re-check itself. Done on the + * spot, that re-check comes straight back: a shortage that lasts turns + * the engine thread into a tight epoll_wait/sendmsg/epoll_ctl loop, + * which is the worst moment to burn a core on. The timer paces it + * instead -- the retry still happens, just not as fast as the kernel can + * refuse it. See issue #407. + * + * ->retry_delay is what the next arming waits. It doubles up to a cap + * and a successful send puts it back to zero. + * + * ->retry_timer.enabled says the timer is holding a port reference: set + * by nxt_timer_add(), left set while the expired timer waits in the work + * queue, and cleared by nxt_timer_handler() on the line before it calls + * nxt_port_retry_handler(). Both fields are touched on port->engine + * only. + */ + nxt_timer_t retry_timer; + nxt_msec_t retry_delay; + nxt_buf_t *free_bufs; nxt_socket_t pair[2]; @@ -619,6 +643,15 @@ void nxt_port_test_run_read_msg_process(nxt_task_t *task, nxt_port_t *port, * observing peers; counting the call itself distinguishes them. */ NXT_EXPORT extern nxt_uint_t nxt_port_test_broadcasts; + +/* + * Counts entries into the event-loop pass of nxt_port_write_msgs() -- the + * dispatches of the port's write handler. A retry that is not paced makes + * this climb with the number of poll passes rather than with elapsed time, + * which is the whole of what the backoff is there to stop and is not + * observable from the port's state afterwards. + */ +NXT_EXPORT extern nxt_uint_t nxt_port_test_write_dispatches; #endif nxt_inline nxt_int_t diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index ddf718fb6..d4265df9f 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -14,6 +14,18 @@ (int) (NXT_PORT_QUEUE_MSG_SIZE - sizeof(nxt_port_msg_t)) +/* + * The first and the last delay a paced retry waits, in milliseconds. The + * first is short because most shortages are: the common case is one failed + * send and then a retry that works, and a millisecond costs that case + * nothing measurable. The cap is what a shortage that lasts settles at, and + * is the retry latency this trades for not spinning -- see the comment on + * nxt_port_retry_later(). + */ +#define NXT_PORT_RETRY_MIN_DELAY 1 +#define NXT_PORT_RETRY_MAX_DELAY 32 + + static nxt_bool_t nxt_port_can_enqueue_buf(nxt_buf_t *b); static uint8_t nxt_port_enqueue_buf(nxt_task_t *task, nxt_port_msg_t *pm, void *qbuf, nxt_buf_t *b); @@ -43,8 +55,10 @@ static void nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg); static nxt_buf_t *nxt_port_buf_alloc(nxt_port_t *port); static void nxt_port_buf_free(nxt_port_t *port, nxt_buf_t *b); -static void nxt_port_announce(nxt_task_t *task, nxt_port_t *port); +static nxt_bool_t nxt_port_announce(nxt_task_t *task, nxt_port_t *port); static void nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port); +static void nxt_port_retry_later(nxt_task_t *task, nxt_port_t *port); +static void nxt_port_retry_handler(nxt_task_t *task, void *obj, void *data); static void nxt_port_rearm_work_handler(nxt_task_t *task, void *obj, void *data); static void nxt_port_error_handler(nxt_task_t *task, void *obj, void *data); @@ -54,6 +68,8 @@ static nxt_uint_t nxt_port_test_msg_alloc_failure_count; nxt_uint_t nxt_port_test_broadcasts; +nxt_uint_t nxt_port_test_write_dispatches; + void nxt_port_test_msg_alloc_failures(nxt_uint_t failures) @@ -689,9 +705,16 @@ nxt_port_fd_disable_write(nxt_task_t *task, nxt_port_t *port, void *data) * point, because this runs after an allocation failure. The flag is cleared * only once the marker has left: a peer that reads one drains the whole ring, * so anything enqueued in between is covered by the same marker. + * + * Answers whether the marker was attempted and did not go out, which is the + * caller's cue to pace the next attempt rather than make it at once. A port + * that owes nothing, one that can no longer be written to, and an EAGAIN all + * answer 0: the first two have nothing to pace, and after EAGAIN the socket + * is full, so the peer's next read raises an edge and paces the retry by + * itself. */ -static void +static nxt_bool_t nxt_port_announce(nxt_task_t *task, nxt_port_t *port) { ssize_t n; @@ -709,7 +732,7 @@ nxt_port_announce(nxt_task_t *task, nxt_port_t *port) */ if (port->announce == 0 || port->pair[1] == -1) { - return; + return 0; } /* nxt_socketpair_send() reads both, whether or not it sends them. */ @@ -731,9 +754,31 @@ nxt_port_announce(nxt_task_t *task, nxt_port_t *port) if (n == (ssize_t) sizeof(nxt_port_msg_t)) { nxt_atomic_fetch_add(&port->announce, -1); + /* + * The shortage is over, as after a queued send that went out. This + * runs on port->engine, from nxt_port_rearm_now(), so the reset + * keeps the ownership rule the one in nxt_port_write_msgs() keeps. + */ + + port->retry_delay = 0; + nxt_debug(task, "port{%d,%d} %d: queue announced", (int) port->pid, (int) port->id, port->socket.fd); + + return 0; } + + /* + * port->socket.error is this send's and not a stale one: the tests above + * guarantee the send ran, and nxt_socketpair_send() writes the field on + * every outcome. + */ + + if (n == NXT_AGAIN && port->socket.error == NXT_EAGAIN) { + return 0; + } + + return 1; } @@ -756,9 +801,127 @@ nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port) return; } + /* + * A paced retry already owes this port both the enable and the marker: + * the timer ends here. Enabling now would bring the write handler back + * on a socket that stayed writable, which is the spin the timer is there + * to stop, and it would also leave the event active under the timer, so + * the timer's own enable could not force a fresh readiness check. The + * new message waits at most the current delay, as a paced retry does. + */ + + if (port->retry_timer.enabled) { + return; + } + + /* + * A marker that could not be written is the second way this port can + * spin: nxt_port_announce() clears ->announce only on a send that fully + * succeeded, and an enabled event brings the write handler straight + * back, which finds nothing to send, disables the event, and re-arms on + * the owed marker again. Pacing it here rather than at that caller + * covers every one of them, since this is the only place the marker is + * ever attempted. The marker goes first so that a failed one leaves + * the event down, instead of an enable that the retry takes back in the + * same pass -- two epoll_ctl() calls that cancel out. + */ + + if (nxt_slow_path(nxt_port_announce(task, port))) { + nxt_port_retry_later(task, port); + return; + } + nxt_fd_event_enable_write(task->thread->engine, &port->socket); +} + - nxt_port_announce(task, port); +/* + * Arm the retry instead of making it now. + * + * The event is disabled and a timer re-arms it after a delay. What that + * buys is in the comment on ->retry_timer in src/nxt_port.h; what it costs + * is retry latency, up to NXT_PORT_RETRY_MAX_DELAY once a shortage has + * lasted long enough for the delay to reach the cap. That is a deliberate + * loss of immediacy on a path #393 made immediate on purpose, and the trade + * is against burning a core in a tight retry loop at the one moment the + * machine has nothing to spare. + * + * Disabling, rather than blocking, for the reason given above + * nxt_port_fd_disable_write(): a registration left in place reports nothing + * more on a socket that never stopped being writable, so the re-arm has to + * add it back. That function also carries the test for a caller that has + * disabled the event already. + * + * Callable on port->engine only. Both callers are there: the drained pass + * of nxt_port_write_msgs() runs on the port's own engine, and + * nxt_port_rearm_now() is what nxt_port_rearm() posts to it. That is what + * makes the timer safe without a lock. + */ + +static void +nxt_port_retry_later(nxt_task_t *task, nxt_port_t *port) +{ + nxt_event_engine_t *engine; + + if (port->retry_timer.enabled || port->pair[1] == -1) { + return; + } + + engine = port->engine; + + nxt_port_fd_disable_write(task, port, NULL); + + port->retry_delay = (port->retry_delay == 0) + ? NXT_PORT_RETRY_MIN_DELAY + : nxt_min(port->retry_delay * 2, + NXT_PORT_RETRY_MAX_DELAY); + + port->retry_timer.work_queue = &engine->fast_work_queue; + port->retry_timer.handler = nxt_port_retry_handler; + port->retry_timer.task = &engine->task; + port->retry_timer.log = engine->task.log; + + /* + * The reference is what keeps the port -- and the timer, which lives in + * it -- alive until the handler runs. It is also the whole of the close + * handling: an armed timer cannot be cancelled from another engine, and + * cancelling it here would leave nxt_timer_handler() with a pointer into + * a pool this drop had just released. So nothing cancels it. The timer + * fires within the cap whatever became of the port, finds a closed one + * by its pair[1] and does nothing with it, and drops this reference. + * + * The one case this does not cover is an engine that is freed with the + * timer still in its tree: the handler never runs, and the port leaks + * with this reference. That is a leak, not a use-after-free, and an + * engine is freed only when its thread exits (a router thread that a + * lower "listen_threads" removed) or when the process does. + */ + + nxt_port_inc_use(port); + + nxt_timer_add(engine, &port->retry_timer, port->retry_delay); + + nxt_debug(task, "port{%d,%d} %d: retry in %M ms", (int) port->pid, + (int) port->id, port->socket.fd, port->retry_delay); +} + + +static void +nxt_port_retry_handler(nxt_task_t *task, void *obj, void *data) +{ + nxt_port_t *port; + + port = nxt_timer_data(obj, nxt_port_t, retry_timer); + + nxt_port_rearm_now(task, port); + + /* + * Last, because nxt_port_rearm_now() may have armed the timer again -- + * on its own reference, which is why this one can be the last without + * taking the timer down with it. + */ + + nxt_port_use(task, port, -1); } @@ -869,7 +1032,7 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, ssize_t n; nxt_int_t ret; uint32_t mmsg_buf[3 * NXT_IOBUF_MAX * 10]; - nxt_bool_t failed, block_write, enable_write; + nxt_bool_t failed, block_write, enable_write, retry_later; nxt_port_t *port; struct iovec iov[NXT_IOBUF_MAX * 10]; nxt_work_queue_t *wq; @@ -883,10 +1046,17 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, failed = 0; block_write = 0; enable_write = 0; + retry_later = 0; use_delta = 0; wq = &task->thread->engine->fast_work_queue; +#if (NXT_TESTS) + if (data == NULL) { + nxt_port_test_write_dispatches++; + } +#endif + do { if (data) { msg = data; @@ -947,6 +1117,20 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, n = nxt_socketpair_send(&port->socket, msg->fd, iov, sb.niov + 1); if (n > 0) { + /* + * The shortage, if there was one, is over: start the next one + * from the short delay again rather than from wherever the last + * one left off. On the drained pass alone, because that is the + * one that runs on port->engine, and ->retry_delay is owned by + * that engine. An inline send is a poor signal anyway: it can + * come from any thread at any moment, including between two + * retries that are both about to fail. + */ + + if (data == NULL) { + port->retry_delay = 0; + } + if (nxt_slow_path((size_t) n != sb.size + iov[0].iov_len)) { nxt_alert(task, "port %d: short write: %z instead of %uz", port->socket.fd, n, sb.size + iov[0].iov_len); @@ -1065,11 +1249,17 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, * Force the readiness re-check the drained pass does -- disable, * then enable -- so the poller reports the socket writable on * the spot. One epoll_ctl() pair per failed retry. + * + * Paced, though, rather than on the spot. The socket is + * writable, so the re-check comes back at once and the retry + * fails again while the shortage lasts: unpaced, this is a tight + * loop on the engine thread for as long as the machine is short + * of memory (#407). nxt_port_retry_later() does the disable as + * well, so block_write is not set here. */ if (data == NULL && port->socket.error != NXT_EAGAIN) { - block_write = 1; - enable_write = 1; + retry_later = 1; } if (msg->link.next == NULL) { @@ -1140,22 +1330,24 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, } /* - * An owed marker is a reason to come back even when the event needs no - * arming: the first retry runs from the re-arm handler, which enables the - * event, so nothing would set enable_write again and a second ENOBUFS - * would strand the marker. Every later pass through this function tries - * once more. + * Not on a port whose send failed outright: the event-loop pass and the + * inline one both reach this through "fail:", and neither has anything + * to gain from arming a socket that has just refused to carry a message. + * + * A paced retry answers for the owed marker as well -- the timer ends in + * nxt_port_rearm_now(), which sends it -- and arming both would put the + * write event back up at once and undo the pacing. Otherwise an owed + * marker is a reason to come back even when the event needs no arming, + * since the re-arm handler is the only place that attempts it. */ - /* - * Not on a port whose send failed outright, and that covers the - * event-loop pass as well as the inline one: both reach this through - * the same "fail:" label, and neither has anything to gain from arming - * a socket that has just refused to carry a message. - */ + if (!failed) { + if (retry_later) { + nxt_port_retry_later(task, port); - if (!failed && (enable_write || port->announce != 0)) { - nxt_port_rearm(task, port); + } else if (enable_write || port->announce != 0) { + nxt_port_rearm(task, port); + } } if (use_delta != 0) { diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index b1212f018..2202deee8 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -24,6 +24,9 @@ static nxt_int_t nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_enqueue(nxt_task_t *task, nxt_port_t *port, nxt_mp_t *mp, nxt_err_t err, nxt_uint_t fails, nxt_bool_t no_memory, nxt_bool_t *queued); +static void nxt_port_fail_test_turn(nxt_event_engine_t *engine, nxt_msec_t ms); +static nxt_uint_t nxt_port_fail_test_spin(nxt_task_t *task, + nxt_msec_t window, nxt_port_t *rearm); #endif static nxt_int_t nxt_port_fail_test_dead_peer(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr); @@ -2368,6 +2371,172 @@ nxt_port_fail_test_enqueue(nxt_task_t *task, nxt_port_t *port, nxt_mp_t *mp, return NXT_OK; } + +/* + * One turn of the event loop, as nxt_event_engine_start() runs it, except + * that the clock moves on by "ms" instead of being read from the machine. + * + * The backoff arms a timer on the engine, and a leg that measures pacing has + * to be able to tell a retry that came back because time passed from one + * that came back because nothing held it. A real clock cannot answer that + * without sleeping, and sleeping would make the numbers the scheduler's + * rather than the port's. engine->timers.now starts at zero on a fresh + * engine and nxt_timer_expire() is what advances it, so the legs below own + * the clock outright. + */ + +static void +nxt_port_fail_test_turn(nxt_event_engine_t *engine, nxt_msec_t ms) +{ + /* + * Commits pending timer changes and sets timers->minimum, as the loop + * does before it polls; without it nxt_timer_expire() returns early. + */ + + (void) nxt_timer_find(engine); + + engine->event.poll(engine, 0); + + nxt_timer_expire(engine, engine->timers.now + ms); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); +} + + +/* + * Poll hard for "window" virtual milliseconds and answer how many times the + * port's write handler was dispatched. + * + * The inner loop is the busy spin itself: on a socket that stayed writable, + * a retry that re-checks readiness on the spot has epoll_wait() return + * immediately every time, so the engine thread runs this loop as fast as it + * can rather than at any rate the clock sets. Sixteen polls per virtual + * millisecond stands in for that -- the real thing manages far more -- and + * makes the two outcomes differ by a factor, not by a margin: unpaced, the + * count follows the number of polls; paced, it follows the window. + * + * A non-NULL "rearm" is re-armed once every virtual millisecond, as a + * sender on this port would while the shortage lasts: each of its sends + * fails inline too and ends in nxt_port_rearm(). + */ + +static nxt_uint_t +nxt_port_fail_test_spin(nxt_task_t *task, nxt_msec_t window, + nxt_port_t *rearm) +{ + nxt_uint_t i, j; + nxt_event_engine_t *engine; + + engine = task->thread->engine; + + nxt_port_test_write_dispatches = 0; + + for (i = 0; i < window; i++) { + + if (rearm != NULL) { + nxt_port_rearm(task, rearm); + } + + for (j = 0; j < 16; j++) { + engine->event.poll(engine, 0); + + nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + } + + nxt_port_fail_test_turn(engine, 1); + } + + return nxt_port_test_write_dispatches; +} + + +/* + * Spin for 64 virtual milliseconds with the failure held, report the count, + * and require it to follow the clock rather than the polls: sixteen is one + * dispatch per virtual millisecond, which no paced retry reaches and every + * unpaced one exceeds by the poll factor. + */ + +static nxt_int_t +nxt_port_fail_test_paced(nxt_task_t *task, nxt_uint_t leg, const char *what, + nxt_port_t *rearm) +{ + nxt_uint_t dispatches; + + dispatches = nxt_port_fail_test_spin(task, 64, rearm); + + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: %s, %ui write dispatches in a " + "64ms window of held ENOMEM", leg, what, dispatches); + + if (dispatches > 16) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: the write handler was " + "dispatched %ui times in a 64ms window of held ENOMEM, " + "expected at most 16; the retry is not paced", + leg, dispatches); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Let the injection go and require the marker to arrive anyway, once: + * pacing a retry must not turn into dropping one. The paced retry is + * waited for on the clock, since nothing else will bring it back. + */ + +static nxt_int_t +nxt_port_fail_test_recovers(nxt_task_t *task, nxt_uint_t leg, + nxt_port_t *port, nxt_fd_t peer) +{ + u_char block[sizeof(nxt_port_msg_t) + 1]; + ssize_t n; + nxt_uint_t i; + nxt_event_engine_t *engine; + + engine = task->thread->engine; + + nxt_socketpair_test_send_fail(0, 0); + + for (i = 0; i < 8; i++) { + if (port->announce == 0 && nxt_queue_is_empty(&port->messages)) { + break; + } + + nxt_port_fail_test_turn(engine, 64); + } + + n = recv(peer, block, sizeof(block), MSG_DONTWAIT); + + if (n != (ssize_t) sizeof(nxt_port_msg_t) + || ((nxt_port_msg_t *) block)->type != _NXT_PORT_MSG_READ_QUEUE) + { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: the peer read %z bytes " + "once the shortage was over, expected a %uz byte " + "READ_QUEUE header; the paced retry never came back " + "(write event state %d)", leg, n, + sizeof(nxt_port_msg_t), (int) port->socket.write); + return NXT_ERROR; + } + + if (!nxt_queue_is_empty(&port->messages) || port->announce != 0 + || nxt_port_fail_test_completions != 1) + { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: after the marker went out " + "announce is %d and the payload was completed %ui " + "times, expected 0 and once", leg, + (int) port->announce, nxt_port_fail_test_completions); + return NXT_ERROR; + } + + return NXT_OK; +} + #endif @@ -2589,6 +2758,11 @@ nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) * coming either. The retry pass has to force the readiness re-check * itself; without it the marker, and every message queued behind it, * waits for ever. + * + * That re-check is paced (#407), so the marker does not go out on the + * next poll but on the first one after the timer the retry armed has + * expired. Which is why the legs from here on drive the clock as well + * as the poller. */ engine->event.poll(engine, 0); @@ -2624,11 +2798,15 @@ nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) goto done; } - /* The second poll has only the re-check to bring the retry back. */ + /* + * The re-check is all there is to bring the retry back, and it is armed + * on a timer: the first turn expires it and re-arms the write event, the + * second is the poll that then finds the socket writable. + */ - engine->event.poll(engine, 0); + nxt_port_fail_test_turn(engine, 64); - nxt_port_fail_test_drain_wq(&engine->fast_work_queue); + nxt_port_fail_test_turn(engine, 0); nxt_socketpair_test_send_fail(0, 0); @@ -2729,6 +2907,144 @@ nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) goto done; } + /* + * Leg 6: an ENOMEM that does not let up, on a queued message. + * + * The retry has to keep coming back -- that is #392 and #393 -- but a + * shortage lasts, and the socket it cannot send on stays writable, so a + * retry that re-checks readiness on the spot is dispatched again the + * moment it returns. The engine thread then spends a core on + * epoll_wait/sendmsg/epoll_ctl for as long as the machine is short of + * memory (#407). + * + * So: hold the failure, poll far harder than any clock would justify, + * and count the dispatches. Unpaced the count is the poll count; paced + * it is a handful, set by the window and the delays. Then let the + * injection go and require the marker to arrive anyway: pacing a retry + * must not turn into dropping one. + */ + + nxt_port_fail_test_turn(engine, 0); + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 0, + NULL) + != NXT_OK) + { + goto done; + } + + if (nxt_port_fail_test_paced(task, 6, "queued marker", NULL) != NXT_OK) { + goto done; + } + + if (nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 6: the marker was dropped " + "while the retry was being paced"); + goto done; + } + + if (nxt_port_fail_test_recovers(task, 6, port, pair[0]) != NXT_OK) { + goto done; + } + + /* + * Leg 7: the same shortage on the owed marker, which is the other way + * this port can spin and does not come from #393 at all. + * nxt_port_announce() clears port->announce only on a send that fully + * succeeded, and the re-arm attempts the marker every pass, so a marker + * that keeps failing spins whether or not the queued path is paced. + * + * Here the marker is owed rather than queued, because the allocation + * that would have held it failed too -- leg 4's setup, with a failure + * that does not stop after one. + */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 1, + NULL) + != NXT_OK) + { + goto done; + } + + if (port->announce == 0 || !nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 7: the marker is neither owed " + "nor queued (announce %d)", (int) port->announce); + goto done; + } + + if (nxt_port_fail_test_paced(task, 7, "owed marker", NULL) != NXT_OK) { + goto done; + } + + if (port->announce == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 7: the owed marker was " + "forgotten while the retry was being paced"); + goto done; + } + + if (nxt_port_fail_test_recovers(task, 7, port, pair[0]) != NXT_OK) { + goto done; + } + + /* + * The shortage is over, so the next one has to start from the short + * delay again. The marker went out through nxt_port_announce(), not + * through a queued send, and only a queued send used to reset it. + */ + + if (port->retry_delay != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 7: the retry delay is %d ms " + "after the owed marker went out, expected 0", + (int) port->retry_delay); + goto done; + } + + /* + * Leg 8: leg 7's shortage, with senders that keep arriving. + * + * Each of their sends fails inline too and re-arms the port. A re-arm + * that enables the write event while the retry timer is armed brings + * the handler back on a socket that stayed writable, and the handler + * re-arms on the owed marker again: leg 7's spin, for as long as the + * senders keep coming. + */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 1, + NULL) + != NXT_OK) + { + goto done; + } + + if (port->announce == 0 || !port->retry_timer.enabled) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 8: the marker is not owed or " + "the retry is not armed (announce %d, armed %d)", + (int) port->announce, (int) port->retry_timer.enabled); + goto done; + } + + if (nxt_port_fail_test_paced(task, 8, "owed marker re-armed every ms", + port) + != NXT_OK) + { + goto done; + } + + if (nxt_port_fail_test_recovers(task, 8, port, pair[0]) != NXT_OK) { + goto done; + } + ret = NXT_OK; done: From 593f9f5c641dc736e37ac649feebfebe8a65fb34 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 23:13:08 +0000 Subject: [PATCH 283/391] fix(router): refuse request lengths that do not fit the app protocol nxt_router_prepare_msg() builds the nxt_unit_request_t for the application in shared memory. Several lengths land in uint8_t fields of the libunit protocol and were stored with a plain assignment, while the full string was still copied. The application then read a different string from the one copied, with the NUL somewhere else: - the HTTP parser does not bound the method, so a method of 256 bytes arrived with method_length 0, and one of 280 bytes with 24; - a field name is capped at 255 bytes by the parser, but the PHP, Perl and Ruby modules prepend "HTTP_", so names of 251 to 255 bytes arrived with name_length 0 to 4. A client can send both. Check every such length against UINT8_MAX before the buffer is allocated. A method that does not fit answers 501, a field name that does not fit with the prefix answers 431, both logged at info. The version and the address and port texts are produced by the router itself and fail with 500. nxt_router_prepare_msg() now reports the status to its caller. Tests: src/test/nxt_router_prepare_msg_test.c drives the function with a hand-built request and checks the stored lengths or the status; test/test_php_protocol_lengths.py checks 431 and 501 end to end. This is the request side of 338a2d49, split out of https://github.com/freeunitorg/freeunit/pull/446 so it can land before the hardening stack. The plain UINT8_MAX checks need neither freeunitorg/freeunit#444 nor freeunitorg/freeunit#445. Co-Authored-By: Claude Opus 5.5 --- auto/sources | 1 + src/nxt_router.c | 101 ++++++++++- src/test/nxt_router_prepare_msg_test.c | 235 +++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/test_php_protocol_lengths.py | 29 +++ 6 files changed, 362 insertions(+), 9 deletions(-) create mode 100644 src/test/nxt_router_prepare_msg_test.c create mode 100644 test/test_php_protocol_lengths.py diff --git a/auto/sources b/auto/sources index 6e0b9fcb9..61140a5db 100644 --- a/auto/sources +++ b/auto/sources @@ -192,6 +192,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_rpc_fd_test.c \ src/test/nxt_port_queued_fd_test.c \ src/test/nxt_conn_close_test.c \ + src/test/nxt_router_prepare_msg_test.c \ " diff --git a/src/nxt_router.c b/src/nxt_router.c index d99010e97..1cb545352 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -380,7 +380,8 @@ static void nxt_router_http_request_done(nxt_task_t *task, void *obj, static void nxt_router_app_prepare_request(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data); static nxt_buf_t *nxt_router_prepare_msg(nxt_task_t *task, - nxt_http_request_t *r, nxt_app_t *app, const nxt_str_t *prefix); + nxt_http_request_t *r, nxt_app_t *app, const nxt_str_t *prefix, + nxt_http_status_t *status); static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data); static void nxt_router_adjust_idle_timer(nxt_task_t *task, void *obj, @@ -7518,6 +7519,7 @@ nxt_router_app_prepare_request(nxt_task_t *task, nxt_buf_t *buf, *body; nxt_int_t res; nxt_port_t *port, *reply_port; + nxt_http_status_t status; int notify; struct { @@ -7538,13 +7540,14 @@ nxt_router_app_prepare_request(nxt_task_t *task, reply_port = task->thread->engine->port; buf = nxt_router_prepare_msg(task, req_rpc_data->request, app, - nxt_app_msg_prefix[app->type]); + nxt_app_msg_prefix[app->type], &status); if (nxt_slow_path(buf == NULL)) { - nxt_alert(task, "stream #%uD, app '%V': failed to prepare app message", - req_rpc_data->stream, &app->name); + if (status == NXT_HTTP_INTERNAL_SERVER_ERROR) { + nxt_alert(task, "stream #%uD, app '%V': failed to prepare app " + "message", req_rpc_data->stream, &app->name); + } - nxt_http_request_error(task, req_rpc_data->request, - NXT_HTTP_INTERNAL_SERVER_ERROR); + nxt_http_request_error(task, req_rpc_data->request, status); return; } @@ -7612,9 +7615,32 @@ nxt_router_app_prepare_request(nxt_task_t *task, +/* + * Builds the nxt_unit_request_t for the application in shared memory. + * + * Every length that lands in a narrow field of the libunit protocol is + * checked before the buffer is allocated, and the request is refused rather + * than having the field truncated: a truncated length makes the application + * see a different string from the one that was copied (a 256-byte method + * arrived as an empty one), with the NUL terminator somewhere else. + * + * - method_length is uint8_t, and the HTTP parser does not bound a method + * (it is limited only by the header buffer): 501. + * - a field's name_length is uint8_t. nxt_http_parse_field_name() caps a + * name at 255 bytes (NXT_HTTP_MAX_FIELD_NAME), which fits alone, but the + * PHP/Perl/Ruby prefix "HTTP_" is added on top: names of 251..255 bytes + * wrapped to 0..4. 431. + * - version, the address and port texts are uint8_t as well but produced + * by Unit itself; they are checked all the same, and fail with 500. + * - the uint32_t lengths (server name, target, path, query, values) are + * bounded by req_size <= PORT_MMAP_DATA_SIZE. + * + * On failure NULL is returned and *status says what to answer. + */ + static nxt_buf_t * nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, - nxt_app_t *app, const nxt_str_t *prefix) + nxt_app_t *app, const nxt_str_t *prefix, nxt_http_status_t *status) { void *target_pos, *query_pos; u_char *pos, *end, *p, c; @@ -7627,6 +7653,29 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, nxt_http_fields_iter_t iter, dup_iter; nxt_unit_request_t *req; + *status = NXT_HTTP_INTERNAL_SERVER_ERROR; + + if (nxt_slow_path(r->method->length > UINT8_MAX)) { + nxt_log(task, NXT_LOG_INFO, "request method of %uz bytes is too long " + "for the application protocol", r->method->length); + + *status = NXT_HTTP_NOT_IMPLEMENTED; + return NULL; + } + + if (nxt_slow_path(r->version.length > UINT8_MAX + || r->remote->address_length > UINT8_MAX + || r->local->address_length > UINT8_MAX + || nxt_sockaddr_port_length(r->local) > UINT8_MAX)) + { + nxt_alert(task, "request version or address too long for the " + "application protocol"); + + return NULL; + } + + /* Every length is of data the request holds, so the sum cannot wrap. */ + req_size = sizeof(nxt_unit_request_t) + r->method->length + 1 + r->version.length + 1 @@ -7645,6 +7694,15 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, { fields_count++; + if (nxt_slow_path(field->name_length + prefix->length > UINT8_MAX)) { + nxt_log(task, NXT_LOG_INFO, "header field name of %d bytes is " + "too long for the application protocol with prefix " + "\"%V\"", (int) field->name_length, prefix); + + *status = NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE; + return NULL; + } + req_size += field->name_length + prefix->length + 1 + field->value_length + 1; } nxt_http_fields_loop; @@ -7652,8 +7710,8 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, req_size += fields_count * sizeof(nxt_unit_field_t); if (nxt_slow_path(req_size > PORT_MMAP_DATA_SIZE)) { - nxt_alert(task, "headers to big to fit in shared memory (%d)", - (int) req_size); + nxt_alert(task, "headers too big to fit in shared memory (%uz)", + req_size); return NULL; } @@ -7755,6 +7813,7 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, dst_field->hash = field->hash; dst_field->skip = 0; + /* Checked to fit uint8_t when req_size was summed. */ dst_field->name_length = field->name_length + prefix->length; dst_field->value_length = field->value_length; @@ -7896,6 +7955,30 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, } +#if (NXT_TESTS) + +/* + * For src/test/nxt_router_prepare_msg_test.c, which repeats this prototype: + * nxt_router.h cannot name nxt_http_status_t. + */ + +nxt_buf_t *nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, + nxt_app_t *app, nxt_bool_t use_http_prefix, nxt_http_status_t *status); + + +nxt_buf_t * +nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, + nxt_app_t *app, nxt_bool_t use_http_prefix, nxt_http_status_t *status) +{ + return nxt_router_prepare_msg(task, r, app, + use_http_prefix ? &http_prefix + : &empty_prefix, + status); +} + +#endif + + static void nxt_router_app_timeout(nxt_task_t *task, void *obj, void *data) { diff --git a/src/test/nxt_router_prepare_msg_test.c b/src/test/nxt_router_prepare_msg_test.c new file mode 100644 index 000000000..8939ce1c4 --- /dev/null +++ b/src/test/nxt_router_prepare_msg_test.c @@ -0,0 +1,235 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * nxt_router_prepare_msg() (src/nxt_router.c) stores the method and header + * name lengths as uint8_t. A 251..255-byte name plus the "HTTP_" prefix, + * or a method over 255 bytes, used to wrap and be stored truncated; now they + * are refused with 431 and 501. The limits themselves are pinned. + * + * The request is built by hand with one field; the engine is a stub that + * only provides the memory pool the buffer is allocated from. The cases + * run in a child process: the accepted cases' buffers hold shared memory + * that only a real engine can release, and the child's exit frees it. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +nxt_buf_t *nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, + nxt_app_t *app, nxt_bool_t use_http_prefix, nxt_http_status_t *status); + + +static u_char nxt_prepare_msg_test_name[300]; +static u_char nxt_prepare_msg_test_method[300]; + + +typedef struct { + const char *name; + nxt_bool_t prefix; + size_t method_length; + size_t field_length; + nxt_http_status_t status; /* 0: accepted */ +} nxt_prepare_msg_test_case_t; + + +static const nxt_prepare_msg_test_case_t nxt_prepare_msg_test_cases[] = { + { "short name, prefix", 1, 3, 6, 0 }, + { "250-byte name + prefix = 255", 1, 3, 250, 0 }, + { "251-byte name + prefix", 1, 3, 251, + NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, + { "253-byte name + prefix", 1, 3, 253, + NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, + { "255-byte name + prefix", 1, 3, 255, + NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, + { "255-byte name, no prefix", 0, 3, 255, 0 }, + { "255-byte method", 0, 255, 6, 0 }, + { "256-byte method", 0, 256, 6, + NXT_HTTP_NOT_IMPLEMENTED }, + { "280-byte method", 0, 280, 6, + NXT_HTTP_NOT_IMPLEMENTED }, +}; + + +static nxt_int_t +nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, + nxt_sockaddr_t *sa, const nxt_prepare_msg_test_case_t *tc) +{ + size_t expect; + nxt_buf_t *b; + nxt_str_t *method, *args; + nxt_http_field_t *field; + nxt_unit_field_t *f; + nxt_http_status_t status; + nxt_unit_request_t *req; + nxt_http_request_t *r; + + r = nxt_mp_zalloc(mp, sizeof(nxt_http_request_t)); + method = nxt_mp_zalloc(mp, sizeof(nxt_str_t)); + args = nxt_mp_zalloc(mp, sizeof(nxt_str_t)); + + if (r == NULL || method == NULL || args == NULL) { + return NXT_ERROR; + } + + method->start = nxt_prepare_msg_test_method; + method->length = tc->method_length; + + r->method = method; + nxt_str_set(&r->version, "HTTP/1.1"); + nxt_str_set(&r->server_name, "localhost"); + nxt_str_set(&r->target, "/"); + r->path = &r->target; + r->args = args; + r->remote = sa; + r->local = sa; + r->content_length_n = -1; + + field = &r->inline_fields[0]; + r->num_inline_fields = 1; + + field->hash = 0; + field->name = nxt_prepare_msg_test_name; + field->name_length = tc->field_length; + field->value = (u_char *) "v"; + field->value_length = 1; + + status = 0; + + b = nxt_router_test_prepare_msg(task, r, app, tc->prefix, &status); + + if (b == NULL) { + if (status != tc->status) { + nxt_log_alert(task->log, "prepare msg test \"%s\": refused " + "with %d, expected %d", tc->name, (int) status, + (int) tc->status); + return NXT_ERROR; + } + + return NXT_OK; + } + + req = (nxt_unit_request_t *) b->mem.pos; + f = &req->fields[0]; + + expect = tc->field_length + (tc->prefix ? nxt_length("HTTP_") : 0); + + if (tc->status != 0 + || req->method_length != tc->method_length + || req->fields_count != 1 + || f->name_length != expect + || nxt_strlen(nxt_unit_sptr_get(&f->name)) != expect) + { + nxt_log_alert(task->log, "prepare msg test \"%s\": accepted with " + "method_length %d, name_length %d; expected %d and " + "%uz, or status %d", tc->name, (int) req->method_length, + (int) f->name_length, (int) tc->method_length, expect, + (int) tc->status); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_prepare_msg_test_run(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_str_t addr; + nxt_int_t ret; + nxt_app_t *app; + nxt_uint_t i; + nxt_task_t *task; + nxt_sockaddr_t *sa; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + + task = thr->task; + task->thread = thr; + + nxt_memset(nxt_prepare_msg_test_name, 'a', + sizeof(nxt_prepare_msg_test_name)); + nxt_memset(nxt_prepare_msg_test_method, 'A', + sizeof(nxt_prepare_msg_test_method)); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + app = nxt_mp_zalloc(mp, sizeof(nxt_app_t)); + nxt_str_set(&addr, "127.0.0.1:8080"); + sa = nxt_sockaddr_parse(mp, &addr); + + if (nxt_slow_path(app == NULL || sa == NULL + || nxt_thread_mutex_create(&app->outgoing.mutex) + != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + engine.mem_pool = mp; + engine.task.thread = thr; + engine.task.log = thr->log; + + saved_engine = thr->engine; + thr->engine = &engine; + + ret = NXT_OK; + + for (i = 0; i < nxt_nitems(nxt_prepare_msg_test_cases); i++) { + if (nxt_prepare_msg_test_case(task, mp, app, sa, + &nxt_prepare_msg_test_cases[i]) + != NXT_OK) + { + ret = NXT_ERROR; + } + } + + thr->engine = saved_engine; + + return ret; +} + + +nxt_int_t +nxt_router_prepare_msg_test(nxt_thread_t *thr) +{ + int status; + pid_t child; + + child = fork(); + + if (child == 0) { + _exit(nxt_prepare_msg_test_run(thr) == NXT_OK ? 0 : 1); + } + + if (nxt_slow_path(child == -1 || waitpid(child, &status, 0) != child)) { + nxt_log_alert(thr->log, "router prepare msg test: fork() failed %E", + nxt_errno); + return NXT_ERROR; + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_log_alert(thr->log, "router prepare msg test failed, " + "status %d", status); + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router prepare msg test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 41cd2150d..ced0f35b4 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -258,6 +258,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_prepare_msg_test(thr) != NXT_OK) { + return 1; + } + if (nxt_main_start_process_reply_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index c6cf4c5a3..02ac84b99 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -86,6 +86,7 @@ nxt_int_t nxt_router_app_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_detached_test(nxt_thread_t *thr); nxt_int_t nxt_router_websocket_test(nxt_thread_t *thr); +nxt_int_t nxt_router_prepare_msg_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); diff --git a/test/test_php_protocol_lengths.py b/test/test_php_protocol_lengths.py new file mode 100644 index 000000000..6c0ac040a --- /dev/null +++ b/test/test_php_protocol_lengths.py @@ -0,0 +1,29 @@ +"""Field names and methods too long for the libunit uint8_t lengths are +refused with 431 and 501 instead of reaching PHP truncated.""" + +from unit.applications.lang.php import ApplicationPHP + +prerequisites = {'modules': {'php': 'any'}} + +client = ApplicationPHP() + + +def test_php_protocol_field_name_length(): + client.load('variables') + + # "HTTP_" + name must fit in 255 bytes. + for length, status in ((250, 200), (251, 431), (253, 431), (255, 431)): + resp = client.get(headers={'Host': 'localhost', 'X' * length: 'v'}) + assert resp['status'] == status, length + + assert client.get()['status'] == 200, 'still serving' + + +def test_php_protocol_method_length(): + client.load('variables') + + for length, status in ((255, 200), (256, 501), (280, 501)): + resp = client.http('A' * length, headers={'Host': 'localhost'}) + assert resp['status'] == status, length + + assert client.get()['status'] == 200, 'still serving' From 3edd7f964f256f272deeddac06b206a11bcefec7 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 16:31:44 +0200 Subject: [PATCH 284/391] port: hand a marker refused by a dead peer to the error handler nxt_port_announce() answered "pace me" for every failed send that was not EAGAIN, so a marker refused with EPIPE was treated as a memory shortage. nxt_port_rearm_now() then armed the retry timer and never enabled the write event. The timer fired every 32ms for as long as the port lived, and the messages queued behind the marker never reached the error handler: each of them held a port reference, so the port, its buffers and its descriptors leaked until the process was removed. Before 9829884c the event was enabled first, and the write handler's own EPIPE drained the queue. The result is now three-way. NXT_AGAIN means the kernel could not take the marker for want of memory, and the caller paces. NXT_OK covers a sent marker, nothing owed, and EAGAIN, and the caller enables the event. NXT_ERROR means the socket will not recover: nxt_port_announce() hands the queue to nxt_port_error_handler() with the reference it drops, as the "fail:" path of nxt_port_write_msgs() does, stops owing the marker, and the caller does nothing more. Leg 9 owes a marker, queues another, holds ENOMEM until the retry is paced, then turns the injection into EPIPE and runs the clock. It requires an empty queue, nothing owed, no timer armed, use_count back to 1 and nothing read by the peer. With the old return value it fails with the queue still full and the timer armed. Three checks the legs could not make before: the pacing check requires the retry timer to be armed at the end of the 64ms window, since a dispatch count cannot see an owed marker attempted without the event; the recovery check requires the timer to be down once the marker went out; and the teardown runs one turn past the longest delay after nxt_port_close(), so a leg that fails with the timer armed does not free the engine with the timer in its tree. Comments updated to Simplified Technical English (STE): clarified nxt_port_announce() return values, error handling paths, ENOMEM pacing in nxt_port_write_msgs(), and test recovery and teardown invariants. Debug and release: 0 warnings, ./build/tests exit 0, legs 6 to 9 at 7, 0, 0 and 0 dispatches. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Gemini 3.8 Flash --- src/nxt_port_socket.c | 90 +++++++++++++-------- src/test/nxt_port_fail_test.c | 142 ++++++++++++++++++++++++++++++++-- 2 files changed, 194 insertions(+), 38 deletions(-) diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index d4265df9f..5ba4db8c4 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -55,7 +55,7 @@ static void nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg); static nxt_buf_t *nxt_port_buf_alloc(nxt_port_t *port); static void nxt_port_buf_free(nxt_port_t *port, nxt_buf_t *b); -static nxt_bool_t nxt_port_announce(nxt_task_t *task, nxt_port_t *port); +static nxt_int_t nxt_port_announce(nxt_task_t *task, nxt_port_t *port); static void nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port); static void nxt_port_retry_later(nxt_task_t *task, nxt_port_t *port); static void nxt_port_retry_handler(nxt_task_t *task, void *obj, void *data); @@ -706,15 +706,20 @@ nxt_port_fd_disable_write(nxt_task_t *task, nxt_port_t *port, void *data) * only once the marker has left: a peer that reads one drains the whole ring, * so anything enqueued in between is covered by the same marker. * - * Answers whether the marker was attempted and did not go out, which is the - * caller's cue to pace the next attempt rather than make it at once. A port - * that owes nothing, one that can no longer be written to, and an EAGAIN all - * answer 0: the first two have nothing to pace, and after EAGAIN the socket - * is full, so the peer's next read raises an edge and paces the retry by - * itself. + * Return values: + * + * NXT_AGAIN indicates the kernel rejected the marker due to insufficient + * memory; the caller must delay the next attempt. NXT_OK indicates the + * port owes nothing, the marker was sent, or the write returned EAGAIN. + * After EAGAIN, the socket buffer is full, so the next read by the peer + * triggers a write event. If the port is already closed (pair[1] is -1), + * the function also returns NXT_OK. NXT_ERROR indicates an unrecoverable + * socket error. In that case, the function transfers queued messages to + * nxt_port_error_handler(), and the caller must neither enable the write + * event nor schedule a retry. */ -static nxt_bool_t +static nxt_int_t nxt_port_announce(nxt_task_t *task, nxt_port_t *port) { ssize_t n; @@ -732,7 +737,7 @@ nxt_port_announce(nxt_task_t *task, nxt_port_t *port) */ if (port->announce == 0 || port->pair[1] == -1) { - return 0; + return NXT_OK; } /* nxt_socketpair_send() reads both, whether or not it sends them. */ @@ -765,7 +770,7 @@ nxt_port_announce(nxt_task_t *task, nxt_port_t *port) nxt_debug(task, "port{%d,%d} %d: queue announced", (int) port->pid, (int) port->id, port->socket.fd); - return 0; + return NXT_OK; } /* @@ -774,11 +779,31 @@ nxt_port_announce(nxt_task_t *task, nxt_port_t *port) * every outcome. */ - if (n == NXT_AGAIN && port->socket.error == NXT_EAGAIN) { - return 0; + if (n == NXT_AGAIN) { + return (port->socket.error == NXT_EAGAIN) ? NXT_OK : NXT_AGAIN; } - return 1; + /* + * If the peer closed the socket or an unrecoverable error occurred, + * do not schedule a retry. Delaying retries would leave the timer + * running indefinitely without re-enabling the write event. As a + * result, queued messages would never reach the error handler. + * + * Transfer queued messages to nxt_port_error_handler() now, matching + * the fail path in nxt_port_write_msgs(). Decrement announce by 1; + * any leftover count on a failed port is harmless because NXT_ERROR + * never arms the retry timer. + */ + + nxt_atomic_fetch_add(&port->announce, -1); + + nxt_port_inc_use(port); + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + nxt_port_error_handler, task, &port->socket, + &port->socket); + + return NXT_ERROR; } @@ -797,6 +822,8 @@ nxt_port_announce(nxt_task_t *task, nxt_port_t *port) static void nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port) { + nxt_int_t ret; + if (port->pair[1] == -1) { return; } @@ -823,11 +850,18 @@ nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port) * covers every one of them, since this is the only place the marker is * ever attempted. The marker goes first so that a failed one leaves * the event down, instead of an enable that the retry takes back in the - * same pass -- two epoll_ctl() calls that cancel out. + * same pass -- two epoll_ctl() calls that cancel out. If sending the + * marker fails permanently, nxt_port_announce() hands the port to + * nxt_port_error_handler(), and there is nothing left to arm. */ - if (nxt_slow_path(nxt_port_announce(task, port))) { - nxt_port_retry_later(task, port); + ret = nxt_port_announce(task, port); + + if (nxt_slow_path(ret != NXT_OK)) { + if (ret == NXT_AGAIN) { + nxt_port_retry_later(task, port); + } + return; } @@ -1240,22 +1274,16 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, * NXT_AGAIN. After EAGAIN the socket is full, and the peer's * next read raises the edge that brings this handler back. * After ENOBUFS or ENOMEM it is not full: the kernel could not - * allocate for the call, and no edge is coming. The inline - * pass re-arms a disabled event, which is how the first retry - * runs at all; but this pass finds the event active, and an - * active edge-triggered event is never re-added, so a retry - * that runs out of memory again would leave the message queued - * for ever, and every later message on this port behind it. - * Force the readiness re-check the drained pass does -- disable, - * then enable -- so the poller reports the socket writable on - * the spot. One epoll_ctl() pair per failed retry. + * allocate for the call, and no edge is coming. This pass + * finds the event active, and an active edge-triggered event + * is never re-added, so a retry must force a readiness re-check. * - * Paced, though, rather than on the spot. The socket is - * writable, so the re-check comes back at once and the retry - * fails again while the shortage lasts: unpaced, this is a tight - * loop on the engine thread for as long as the machine is short - * of memory (#407). nxt_port_retry_later() does the disable as - * well, so block_write is not set here. + * The retry is paced rather than attempted on the spot. The + * socket is writable, so an immediate re-check fails again + * while the memory shortage lasts (#407). Instead, + * nxt_port_retry_later() disables the event and sets a timer + * to re-enable it. Because it handles the event disable directly, + * block_write is not set here. */ if (data == NULL && port->socket.error != NXT_EAGAIN) { diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index 2202deee8..ccca0324d 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -2380,9 +2380,9 @@ nxt_port_fail_test_enqueue(nxt_task_t *task, nxt_port_t *port, nxt_mp_t *mp, * to be able to tell a retry that came back because time passed from one * that came back because nothing held it. A real clock cannot answer that * without sleeping, and sleeping would make the numbers the scheduler's - * rather than the port's. engine->timers.now starts at zero on a fresh - * engine and nxt_timer_expire() is what advances it, so the legs below own - * the clock outright. + * rather than the port's. In these tests, only nxt_timer_expire() + * advances engine->timers.now. The test functions control the timer clock + * directly by advancing it in discrete intervals. */ static void @@ -2459,12 +2459,26 @@ nxt_port_fail_test_spin(nxt_task_t *task, nxt_msec_t window, static nxt_int_t nxt_port_fail_test_paced(nxt_task_t *task, nxt_uint_t leg, const char *what, - nxt_port_t *rearm) + nxt_port_t *port, nxt_port_t *rearm) { nxt_uint_t dispatches; dispatches = nxt_port_fail_test_spin(task, 64, rearm); + /* + * Write dispatch counters do not record attempts made without an active + * event. Verify pacing directly: ensure the retry timer remains enabled + * at the end of the 64 ms window. + */ + + if (!port->retry_timer.enabled) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: no retry timer is armed " + "after a 64ms window of held ENOMEM; the retry is not " + "paced", leg); + return NXT_ERROR; + } + nxt_log_error(NXT_LOG_NOTICE, task->log, "port failure test: leg %ui: %s, %ui write dispatches in a " "64ms window of held ENOMEM", leg, what, dispatches); @@ -2534,6 +2548,18 @@ nxt_port_fail_test_recovers(nxt_task_t *task, nxt_uint_t leg, return NXT_ERROR; } + /* + * An active timer retains a port reference. If the timer is not disabled, + * the port leaks when the test frees the engine. + */ + + if (port->retry_timer.enabled) { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test: leg %ui: the retry timer is still " + "armed after the marker went out", leg); + return NXT_ERROR; + } + return NXT_OK; } @@ -2935,7 +2961,9 @@ nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) goto done; } - if (nxt_port_fail_test_paced(task, 6, "queued marker", NULL) != NXT_OK) { + if (nxt_port_fail_test_paced(task, 6, "queued marker", port, NULL) + != NXT_OK) + { goto done; } @@ -2978,7 +3006,9 @@ nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) goto done; } - if (nxt_port_fail_test_paced(task, 7, "owed marker", NULL) != NXT_OK) { + if (nxt_port_fail_test_paced(task, 7, "owed marker", port, NULL) + != NXT_OK) + { goto done; } @@ -3035,7 +3065,7 @@ nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) } if (nxt_port_fail_test_paced(task, 8, "owed marker re-armed every ms", - port) + port, port) != NXT_OK) { goto done; @@ -3045,6 +3075,93 @@ nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) goto done; } + /* + * Leg 9: A send fails with EPIPE while a retry is delayed and a marker + * is pending. + * + * When the retry timer fires, nxt_port_rearm_now() attempts to send + * the pending marker first. + * + * If the send returns EPIPE or another unrecoverable error, the error + * is permanent, not a transient memory shortage. Continuing to delay + * retries would leave the timer running indefinitely without re-enabling + * the write event. In that case, queued messages would never reach the + * error handler, leaking port references. + * + * Instead, nxt_port_announce() must forward queued messages to + * nxt_port_error_handler(), matching the write failure path. + * + * Sequence: + * 1. Trigger a pending marker (simulate ENOMEM when queuing). + * 2. Queue a second marker. + * 3. Hold ENOMEM until the retry timer is set. + * 4. Change simulated error to EPIPE and advance the timer clock. + */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 1, + NULL) + != NXT_OK) + { + goto done; + } + + /* The helper counts one completion per enqueue. */ + + nxt_port_fail_test_completions = 0; + + if (nxt_port_fail_test_enqueue(task, port, mp, NXT_ENOMEM, 100000, 0, + NULL) + != NXT_OK) + { + goto done; + } + + if (port->announce == 0 || nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 9: expected an owed and a " + "queued marker (announce %d, queue empty %d)", + (int) port->announce, + (int) nxt_queue_is_empty(&port->messages)); + goto done; + } + + if (nxt_port_fail_test_paced(task, 9, "owed and queued marker", port, + NULL) + != NXT_OK) + { + goto done; + } + + nxt_socketpair_test_send_fail(NXT_EPIPE, 100000); + + for (i = 0; i < 4; i++) { + nxt_port_fail_test_turn(engine, 64); + } + + n = recv(pair[0], block, sizeof(block), MSG_DONTWAIT); + + if (n != -1 || nxt_errno != NXT_EAGAIN + || !nxt_queue_is_empty(&port->messages) + || port->announce != 0 || port->retry_timer.enabled + || nxt_fd_event_is_active(port->socket.write) + || port->use_count != 1 || nxt_port_fail_test_completions != 1) + { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: leg 9: after the send failed with " + "EPIPE, %d bytes read (errno %d), queue empty %d, " + "announce %d, timer armed %d, write active %d, " + "use_count %d, completions %ui; expected -1, %d, 1, 0, " + "0, 0, 1 and 1", (int) n, (int) nxt_errno, + (int) nxt_queue_is_empty(&port->messages), + (int) port->announce, (int) port->retry_timer.enabled, + (int) nxt_fd_event_is_active(port->socket.write), + (int) port->use_count, nxt_port_fail_test_completions, + (int) NXT_EAGAIN); + goto done; + } + ret = NXT_OK; done: @@ -3060,6 +3177,17 @@ nxt_port_fail_test_wakeup_errno(nxt_thread_t *thr) nxt_port_close(task, port); + /* + * If a test leg fails while the retry timer is enabled, the timer retains + * a port reference. + * + * Advance the event loop past the maximum delay. The expired timer detects + * that the port is closed and releases its reference, allowing clean + * engine teardown. + */ + + nxt_port_fail_test_turn(engine, 64); + if (pair[0] != -1 && nxt_test_fd_is_open(pair[0])) { nxt_fd_close(pair[0]); } From b00ab4c2de48453bfcd19081e1f37252164d8e3a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 17:33:25 +0000 Subject: [PATCH 285/391] main: refuse a WHOAMI that does not match how the sender was started The WHOAMI handler trusted the sender. "ppid" comes from the message body, and nothing stopped a process from sending WHOAMI twice. - A worker that sent WHOAMI again with a port got a second port 0 in main, and was linked into its parent's "children" queue a second time. Without a port it was linked a second time too. A second insert of the same link corrupts the queue in main, which runs as root. - A process could name any recorded process as its parent, for example the router, and be linked into its "children". Only two shapes are valid: a process forked by main sends no port, and a worker forked by a prototype sends its port once. Everything else is refused with an alert, and the descriptor it carried is closed. A debug assert checks that a new child is not linked yet. src/test/nxt_main_whoami_test.c drives the handler with four forged messages and checks that the records stay as they were and the descriptor is closed. Without the fix the first case aborts a debug build ("port #0 for pid ... add failed", then an assertion). Co-Authored-By: Claude Opus 5.5 --- CHANGES | 5 + auto/sources | 1 + docs/changes.xml | 8 + src/nxt_main_process.c | 46 ++++++ src/nxt_main_process.h | 2 + src/test/nxt_main_whoami_test.c | 261 ++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 8 files changed, 328 insertions(+) create mode 100644 src/test/nxt_main_whoami_test.c diff --git a/CHANGES b/CHANGES index da6e7481c..16b765d67 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: the main process accepted a second WHOAMI from an + application process, and a parent that is not a prototype. The + process was then linked into its parent's list twice, which + corrupted the list in main. Such a message is now refused. + *) Bugfix: an upstream 1xx interim response ("100 Continue", "103 Early Hints") was relayed to the client as the response, with the final response relayed as its body. The proxy now drops interim responses diff --git a/auto/sources b/auto/sources index 61140a5db..a2fdccb9a 100644 --- a/auto/sources +++ b/auto/sources @@ -186,6 +186,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ src/test/nxt_proto_creating_wedge_test.c \ + src/test/nxt_main_whoami_test.c \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_ctrunc_test.c \ src/test/nxt_port_fd_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 62b557c15..7fe4c6981 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,14 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +the main process accepted a second WHOAMI from an application process, and a +parent that is not a prototype. The process was then linked into its parent's +list twice, which corrupted the list in main. Such a message is now refused. + + + an upstream 1xx interim response ("100 Continue", "103 Early Hints") was diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 0be5c4635..993d8c0a5 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -758,6 +758,14 @@ nxt_main_test_run_file_store(nxt_task_t *task, const char *dir, return nxt_main_file_store(task, dir, tmp_name, name, buf, size); } + +/* Lets src/test/nxt_main_whoami_test.c drive the WHOAMI handler. */ +void +nxt_main_test_run_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_main_process_whoami_handler(task, msg); +} + #endif @@ -897,6 +905,41 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "whoami: from %PI, parent %PI, fd %d", pid, ppid, msg->fd[0]); + /* + * The sender is not trusted, and "ppid" comes from its message. Only + * two shapes are valid: a process forked by main sends no descriptor, + * and a worker forked by a prototype sends its port. A process sends + * WHOAMI once, so a second one with a port is refused too; without + * that, it would get a second port and be linked into "children" + * twice, which corrupts the queue. + */ + + if (ppid == nxt_pid) { + if (nxt_slow_path(msg->fd[0] != -1)) { + nxt_alert(task, "whoami: process %PI sent a port", pid); + goto fail; + } + + } else { + if (nxt_slow_path(nxt_process_type(pprocess) + != NXT_PROCESS_PROTOTYPE)) + { + nxt_alert(task, "whoami: process %PI named %PI as its parent, " + "which is not a prototype", pid, ppid); + goto fail; + } + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_alert(task, "whoami: worker %PI sent no port", pid); + goto fail; + } + + if (nxt_slow_path(nxt_runtime_port_find(rt, pid, 0) != NULL)) { + nxt_alert(task, "whoami: process %PI sent WHOAMI again", pid); + goto fail; + } + } + if (msg->fd[0] != -1) { port = nxt_runtime_process_port_create(task, rt, pid, 0, NXT_PROCESS_APP); @@ -924,6 +967,9 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } if (ppid != nxt_pid) { + /* A new record: the checks above make it so. */ + nxt_assert(port->process->link.next == NULL); + nxt_queue_insert_tail(&pprocess->children, &port->process->link); } diff --git a/src/nxt_main_process.h b/src/nxt_main_process.h index 7cb4cd351..88fcf0b75 100644 --- a/src/nxt_main_process.h +++ b/src/nxt_main_process.h @@ -39,6 +39,8 @@ void nxt_main_test_run_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); nxt_int_t nxt_main_test_run_file_store(nxt_task_t *task, const char *dir, const char *tmp_name, const char *name, u_char *buf, size_t size); +void nxt_main_test_run_whoami_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); #endif diff --git a/src/test/nxt_main_whoami_test.c b/src/test/nxt_main_whoami_test.c new file mode 100644 index 000000000..3d2534b31 --- /dev/null +++ b/src/test/nxt_main_whoami_test.c @@ -0,0 +1,261 @@ +/* + * Copyright (C) FreeUnit Community + */ + +/* + * The WHOAMI handler in main must refuse a message whose shape does not + * match how the sender was started. "ppid" comes from the sender, which is + * not trusted. + * + * - a worker already recorded sends WHOAMI again, with or without a port: + * before the fix it got a second port and was linked into its parent's + * "children" twice, which corrupts the queue; + * - a process names a parent that is not a prototype; + * - a process names main as its parent and sends a port. + * + * Each refused message must leave the records as they were and close the + * descriptor it carried. The accepted shapes run in every pytest that + * starts an application. + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + + +typedef struct { + nxt_pid_t pid; + nxt_pid_t ppid; + nxt_bool_t with_fd; + const char *label; +} nxt_main_whoami_case_t; + + +static nxt_port_t * +nxt_main_whoami_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_pid_t pid, + nxt_process_type_t type) +{ + nxt_port_t *port; + + port = nxt_runtime_process_port_create(task, rt, pid, 0, type); + if (port != NULL) { + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + } + + return port; +} + + +static void +nxt_main_whoami_port_release(nxt_task_t *task, nxt_port_t *port) +{ + if (port != NULL) { + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + } +} + + +static nxt_uint_t +nxt_main_whoami_count(nxt_queue_t *queue) +{ + nxt_uint_t n; + nxt_queue_link_t *lnk; + + n = 0; + + for (lnk = nxt_queue_first(queue); + lnk != nxt_queue_tail(queue) && n < 100; + lnk = nxt_queue_next(lnk)) + { + n++; + } + + return n; +} + + +static nxt_int_t +nxt_main_whoami_refused(nxt_thread_t *thr, nxt_mp_t *mp, nxt_runtime_t *rt, + nxt_process_t *proto, nxt_process_t *child, nxt_main_whoami_case_t *c) +{ + int pair[2]; + nxt_buf_t *buf; + nxt_process_t *process; + nxt_port_recv_msg_t msg; + + pair[0] = -1; + pair[1] = -1; + + if (c->with_fd && socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0) { + nxt_log_alert(thr->log, "main whoami test: socketpair() failed"); + return NXT_ERROR; + } + + buf = nxt_buf_mem_alloc(mp, sizeof(nxt_pid_t), 0); + if (nxt_slow_path(buf == NULL)) { + return NXT_ERROR; + } + + buf->mem.free = nxt_cpymem(buf->mem.free, &c->ppid, sizeof(nxt_pid_t)); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = buf; + msg.fd[0] = pair[0]; + msg.fd[1] = -1; + msg.port_msg.type = _NXT_PORT_MSG_WHOAMI; + msg.port_msg.pid = c->pid; +#if (NXT_USE_CMSG_PID) + msg.cmsg_pid = c->pid; +#endif + + nxt_main_test_run_whoami_handler(thr->task, &msg); + + nxt_mp_free(mp, buf); + + if (pair[1] != -1) { + nxt_fd_close(pair[1]); + } + + if (c->with_fd && nxt_test_fd_is_open(pair[0])) { + nxt_log_alert(thr->log, "main whoami test: %s: the descriptor was " + "kept", c->label); + nxt_fd_close(pair[0]); + return NXT_ERROR; + } + + if (nxt_main_whoami_count(&proto->children) != 1 + || nxt_main_whoami_count(&child->ports) != 1) + { + nxt_log_alert(thr->log, "main whoami test: %s: %ui children, %ui " + "ports of the worker; expected 1 and 1", c->label, + nxt_main_whoami_count(&proto->children), + nxt_main_whoami_count(&child->ports)); + return NXT_ERROR; + } + + if (c->pid != child->pid) { + process = nxt_runtime_process_find(rt, c->pid); + + if (process != NULL) { + nxt_log_alert(thr->log, "main whoami test: %s: a record was " + "created", c->label); + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +nxt_int_t +nxt_main_whoami_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_pid_t proto_pid, child_pid, router_pid, new_pid; + nxt_uint_t i; + nxt_port_t *main_port, *router_port, *proto_port; + nxt_port_t *child_port; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *proto, *child; + nxt_main_whoami_case_t cases[4]; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main whoami test started"); + + ret = NXT_ERROR; + main_port = NULL; + router_port = NULL; + proto_port = NULL; + child_port = NULL; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + saved_rt = thr->runtime; + thr->runtime = rt; + + proto_pid = nxt_pid + 1; + child_pid = nxt_pid + 2; + router_pid = nxt_pid + 3; + new_pid = nxt_pid + 4; + + main_port = nxt_main_whoami_port(thr->task, rt, nxt_pid, + NXT_PROCESS_MAIN); + router_port = nxt_main_whoami_port(thr->task, rt, router_pid, + NXT_PROCESS_ROUTER); + proto_port = nxt_main_whoami_port(thr->task, rt, proto_pid, + NXT_PROCESS_PROTOTYPE); + child_port = nxt_main_whoami_port(thr->task, rt, child_pid, + NXT_PROCESS_APP); + + if (nxt_slow_path(main_port == NULL || router_port == NULL + || proto_port == NULL || child_port == NULL)) + { + goto done; + } + + proto = proto_port->process; + child = child_port->process; + + /* As an accepted WHOAMI leaves a worker. */ + nxt_queue_insert_tail(&proto->children, &child->link); + + cases[0] = (nxt_main_whoami_case_t) { + child_pid, proto_pid, 1, "second WHOAMI with a port" }; + cases[1] = (nxt_main_whoami_case_t) { + child_pid, proto_pid, 0, "second WHOAMI without a port" }; + cases[2] = (nxt_main_whoami_case_t) { + new_pid, router_pid, 1, "parent is not a prototype" }; + cases[3] = (nxt_main_whoami_case_t) { + new_pid, nxt_pid, 1, "child of main sends a port" }; + + for (i = 0; i < nxt_nitems(cases); i++) { + if (nxt_main_whoami_refused(thr, mp, rt, proto, child, &cases[i]) + != NXT_OK) + { + goto done; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main whoami test passed"); + + ret = NXT_OK; + +done: + + nxt_main_whoami_port_release(thr->task, child_port); + nxt_main_whoami_port_release(thr->task, proto_port); + nxt_main_whoami_port_release(thr->task, router_port); + nxt_main_whoami_port_release(thr->task, main_port); + + thr->runtime = saved_rt; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index ced0f35b4..be33697d8 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -274,6 +274,10 @@ main(int argc, char **argv) return 1; } + if (nxt_main_whoami_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 02ac84b99..42baada4b 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -90,6 +90,7 @@ nxt_int_t nxt_router_prepare_msg_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); +nxt_int_t nxt_main_whoami_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); From 589be19d122abc60dece28d4e9263331335b8857 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 19:12:29 +0000 Subject: [PATCH 286/391] application: report a pid-isolated worker that died before it was created Under "isolation": {"namespaces": {"pid": true}} a worker that died in the CREATING state left the main process holding the record, the port and the descriptor it made at WHOAMI time, until the prototype itself exited. A worker that keeps dying in that window costs main one descriptor per attempt. The prototype cannot send REMOVE_PID for it: the only pid it has for such a worker is the namespace-local one from fork(), and REMOVE_PID is broadcast, so that number would name an unrelated process in every other receiver. Main holds both names. A worker's WHOAMI arrives with the global pid in SCM_CREDENTIALS and with the worker's namespace-local pid in the message header (nxt_pid is still local at that point). Main now keeps the pair in process->parent_ns_pid, and a new message, NXT_PORT_MSG_REMOVE_CHILD_PID, lets the prototype name the dead worker by the local pid. Main looks the name up only among the children of the sender the kernel names, so a prototype can retire its own workers and nothing else, and then tells the router by the global pid, as an ordinary REMOVE_PID would. The header pid is chosen by the sender, so a name a live sibling already holds is refused with a warning, and the new worker stays unnamed. This also happens without a forger, when a new worker gets the local pid of one whose report main has not read yet; the cost is the same leak for that one worker. Both ends refuse 0 or a negative pid, since 0 is how "no name" is stored. Main cannot check that the worker is dead, so a bad prototype can drop main's record of a live worker of its own; it can already do the same with REMOVE_PID outside a pid namespace. Without a sender credential (no NXT_USE_CMSG_PID) the sender pid would be the header pid the sender wrote itself, so the handler is not registered there. Nothing is lost: a pid namespace needs Linux, which has SO_PASSCRED. The message slot is appended to nxt_port_handlers_t, so no existing message type is renumbered. A static assert now pins the index of the last slot, so a slot inserted before it fails the build instead of silently renumbering messages that language modules from another release read. The non-isolated path does not change. The five places that took a process off a "children" queue by hand now call nxt_process_unlink(), which also sets link.next to NULL, the "not linked" mark that the release build of nxt_queue_remove() leaves unset. Closes #310. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 5 + auto/sources | 1 + docs/changes.xml | 9 + src/nxt_application.c | 89 ++++- src/nxt_main_process.c | 192 ++++++++- src/nxt_main_process.h | 6 + src/nxt_port.h | 21 + src/nxt_process.h | 23 ++ src/nxt_runtime.c | 7 +- src/test/nxt_main_remove_child_pid_test.c | 463 ++++++++++++++++++++++ src/test/nxt_proto_creating_wedge_test.c | 38 +- src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 13 files changed, 817 insertions(+), 42 deletions(-) create mode 100644 src/test/nxt_main_remove_child_pid_test.c diff --git a/CHANGES b/CHANGES index 16b765d67..14fd34bd8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: with "isolation": {"namespaces": {"pid": true}}, a worker + that died before it was fully started left its record, port and + descriptor in the main process until the prototype exited. The + prototype now reports such a worker to main, which drops them. + *) Bugfix: the main process accepted a second WHOAMI from an application process, and a parent that is not a prototype. The process was then linked into its parent's list twice, which diff --git a/auto/sources b/auto/sources index a2fdccb9a..b0fd84fbc 100644 --- a/auto/sources +++ b/auto/sources @@ -187,6 +187,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_main_file_store_test.c \ src/test/nxt_proto_creating_wedge_test.c \ src/test/nxt_main_whoami_test.c \ + src/test/nxt_main_remove_child_pid_test.c \ src/test/nxt_port_change_file_test.c \ src/test/nxt_port_ctrunc_test.c \ src/test/nxt_port_fd_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 7fe4c6981..1ed99f059 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,15 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +with "isolation": {"namespaces": {"pid": true}}, a worker that died before it +was fully started left its record, port and descriptor in the main process +until the prototype exited. The prototype now reports such a worker to main, +which drops them. + + + the main process accepted a second WHOAMI from an application process, and a diff --git a/src/nxt_application.c b/src/nxt_application.c index 56c132fe1..d98297ad2 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -60,6 +60,8 @@ static void nxt_proto_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_proto_process_created_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_proto_quit_children(nxt_task_t *task); +static void nxt_proto_report_child_pid(nxt_task_t *task, + nxt_process_t *process, nxt_uint_t level); static void nxt_proto_kill_silent(nxt_task_t *task, void *obj, void *data); static nxt_process_t *nxt_proto_process_find(nxt_task_t *task, nxt_pid_t pid); static void nxt_proto_process_add(nxt_task_t *task, nxt_process_t *process); @@ -1274,18 +1276,14 @@ nxt_proto_sigchld_handler(nxt_task_t *task, void *obj, void *data) * every receiver to remove whatever unrelated process holds that number -- and * a prototype's namespace-local counter climbs with each worker it forks, so * it walks into the range the daemon's own pids occupy. Removing a live - * sibling's ports drops requests, which is worse than the leak, so that case - * is left alone and logged. + * sibling's ports drops requests, which is worse than the leak, so no + * REMOVE_PID is sent there. * - * Closing it belongs on the other side. The prototype cannot map its - * namespace-local pid to a global one, but main can map the other way without - * trusting anybody: it holds the global pid from SCM_CREDENTIALS at WHOAMI - * time, and the last entry of /proc//status NSpid is that process's pid - * in its own namespace (Linux 4.1+). Recording that as a second key, plus a - * REMOVE_PID variant scoped to one prototype's children, would close it with - * no sender-supplied pid to authenticate. msg->port_msg.pid on the WHOAMI - * message happens to carry the same number, but it is chosen by the sender and - * would need the authentication NSpid makes unnecessary. + * So the report goes to main alone, as REMOVE_CHILD_PID carrying the + * namespace-local pid. Main holds both names -- the global pid from + * SCM_CREDENTIALS and the local one from the WHOAMI header -- and resolves + * the report among the sender's own children only. It then tells the router + * by the global pid, so the router sees what a REMOVE_PID would have given it. */ void @@ -1407,10 +1405,10 @@ nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process) */ if (nxt_slow_path(rt->is_pid_isolated)) { - nxt_log(task, level, "app process (isolated %PI) died before it was " - "created; it has no globally valid pid to broadcast, so a " - "record the main or router process may hold for it stays " - "until the prototype exits", process->isolated_pid); + nxt_debug(task, "app process (isolated %PI) died before it was " + "created", process->isolated_pid); + + nxt_proto_report_child_pid(task, process, level); /* No REMOVE_PID will carry it; do not leave it armed. */ process->stream = 0; @@ -1422,6 +1420,64 @@ nxt_proto_child_exited(nxt_task_t *task, nxt_process_t *process) } +/* + * Tell main that a child of this prototype died, by its namespace-local pid. + * The message carries no stream: only the router turns one into an RPC error, + * and this goes to main alone. + * + * Ordering: the worker's WHOAMI and this message go to the same write end of + * main's port socketpair, and this runs only after waitpid() reaped the + * worker, so main reads the WHOAMI first. This holds because main's port has + * no shared-memory queue. + */ + +static void +nxt_proto_report_child_pid(nxt_task_t *task, nxt_process_t *process, + nxt_uint_t level) +{ + nxt_buf_t *buf; + nxt_port_t *main_port; + nxt_runtime_t *rt; + + rt = task->thread->runtime; + + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + if (nxt_slow_path(main_port == NULL)) { + goto fail; + } + + buf = nxt_buf_mem_ts_alloc(task, task->thread->engine->mem_pool, + sizeof(nxt_pid_t)); + if (nxt_slow_path(buf == NULL)) { + goto fail; + } + + buf->mem.free = nxt_cpymem(buf->mem.free, &process->isolated_pid, + sizeof(nxt_pid_t)); + + if (nxt_fast_path(nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_REMOVE_CHILD_PID, + -1, 0, 0, buf) + == NXT_OK)) + { + return; + } + + /* Still ours: the port layer takes the buffer only on NXT_OK. */ + + nxt_work_queue_add(&task->thread->engine->fast_work_queue, + buf->completion_handler, task, buf, buf->parent); + +fail: + + /* Main then keeps the record until this prototype exits (#310). */ + + nxt_log(task, level, "app process (isolated %PI) died before it was " + "created and could not be reported to the main process", + process->isolated_pid); +} + + static nxt_app_module_t * nxt_app_module_load(nxt_task_t *task, const char *name) { @@ -1813,8 +1869,7 @@ nxt_proto_process_remove(nxt_task_t *task, nxt_pid_t pid) process = lhq.value; - nxt_queue_remove(&process->link); - process->link.next = NULL; + nxt_process_unlink(process); break; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 993d8c0a5..ad84cf2cf 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -60,6 +60,12 @@ static void nxt_main_port_modules_handler(nxt_task_t *task, static int nxt_cdecl nxt_app_lang_compare(const void *v1, const void *v2); static void nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +#if (NXT_USE_CMSG_PID) +static void nxt_main_process_name_child(nxt_task_t *task, + nxt_process_t *pprocess, nxt_process_t *process, nxt_pid_t ns_pid); +static void nxt_main_remove_child_pid_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +#endif static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *dir, @@ -766,6 +772,26 @@ nxt_main_test_run_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_main_process_whoami_handler(task, msg); } + +#if (NXT_USE_CMSG_PID) + +void +nxt_main_test_run_name_child(nxt_task_t *task, nxt_process_t *pprocess, + nxt_process_t *process, nxt_pid_t ns_pid) +{ + nxt_main_process_name_child(task, pprocess, process, ns_pid); +} + + +void +nxt_main_test_run_remove_child_pid_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg) +{ + nxt_main_remove_child_pid_handler(task, msg); +} + +#endif + #endif @@ -845,6 +871,9 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { .process_ready = nxt_port_process_ready_handler, .whoami = nxt_main_process_whoami_handler, .remove_pid = nxt_port_remove_pid_handler, +#if (NXT_USE_CMSG_PID) + .remove_child_pid = nxt_main_remove_child_pid_handler, +#endif .start_process = nxt_main_start_process_handler, .socket = nxt_main_port_socket_handler, .socket_unlink = nxt_main_port_socket_unlink_handler, @@ -864,6 +893,59 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { }; +#if (NXT_USE_CMSG_PID) + +/* + * Keep a child's pid in the pid namespace of the prototype that forked it. + * + * Under "isolation": {"namespaces": {"pid": true}} a worker has two pids: the + * global one from SCM_CREDENTIALS, which keys main's record, and the local one + * the prototype got from fork(). The WHOAMI header carries the local one, and + * it is the only name the prototype has for a worker that dies before + * PROCESS_CREATED. nxt_main_remove_child_pid_handler() resolves that name. + * + * The pair is kept even when the two numbers are equal: the counters are + * independent, so equality proves nothing. A name a live sibling still holds + * is refused, and the new worker stays unnamed. This happens without a + * forger too: the prototype forks a new worker with the local pid of one + * whose report main has not read yet. Either way the cost is the #310 leak + * for that one worker. Taking the name over would be worse: a late report + * for the old worker would then remove the live new one. + */ + +static void +nxt_main_process_name_child(nxt_task_t *task, nxt_process_t *pprocess, + nxt_process_t *process, nxt_pid_t ns_pid) +{ + nxt_process_t *child; + + /* 0 means "no name"; no namespace hands out 0 or a negative pid. */ + + if (nxt_slow_path(ns_pid <= 0)) { + return; + } + + nxt_queue_each(child, &pprocess->children, nxt_process_t, link) { + + if (child != process && child->parent_ns_pid == ns_pid) { + nxt_log(task, NXT_LOG_WARN, "pid %PI inside %PI is still held " + "by process %PI; process %PI stays unnamed", ns_pid, + pprocess->pid, child->pid, process->pid); + + return; + } + + } nxt_queue_loop; + + nxt_debug(task, "process %PI is pid %PI inside %PI", process->pid, + ns_pid, pprocess->pid); + + process->parent_ns_pid = ns_pid; +} + +#endif + + static void nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -971,6 +1053,11 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_assert(port->process->link.next == NULL); nxt_queue_insert_tail(&pprocess->children, &port->process->link); + +#if (NXT_USE_CMSG_PID) + nxt_main_process_name_child(task, pprocess, port->process, + msg->port_msg.pid); +#endif } buf = nxt_buf_mem_alloc(task->thread->engine->mem_pool, @@ -1003,6 +1090,105 @@ nxt_main_process_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +#if (NXT_USE_CMSG_PID) + +/* + * A prototype reports a worker of its own that died before PROCESS_CREATED, + * by the worker's namespace-local pid. REMOVE_PID cannot carry that pid: it + * is broadcast, and elsewhere the number names an unrelated process. + * + * The sender is the pid from SCM_CREDENTIALS, and the name is looked up among + * that sender's own children only, so a prototype can retire its own workers + * and nothing else. Main then tells the router by the global pid, as a + * REMOVE_PID would. Main cannot check that the worker is really dead, so a + * bad prototype can drop the record of a live worker of its own; it can do + * the same with REMOVE_PID outside a pid namespace. + * + * Without a sender credential the handler is not registered at all: a pid + * namespace needs Linux, which has SO_PASSCRED. + */ + +static void +nxt_main_remove_child_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + size_t size; + nxt_buf_t *buf; + nxt_pid_t pid, sender; + nxt_runtime_t *rt; + nxt_process_t *pprocess, *child; + + buf = msg->buf; + size = (buf != NULL) ? (size_t) nxt_buf_used_size(buf) : 0; + + if (nxt_slow_path(size != sizeof(nxt_pid_t))) { + nxt_log(task, NXT_LOG_WARN, "REMOVE_CHILD_PID with a %uz byte " + "payload", size); + goto done; + } + + nxt_memcpy(&pid, buf->mem.pos, sizeof(nxt_pid_t)); + + /* 0 is how "no name" is stored, so it must not match anything. */ + + if (nxt_slow_path(pid <= 0)) { + nxt_log(task, NXT_LOG_WARN, "REMOVE_CHILD_PID naming pid %PI", pid); + goto done; + } + + sender = nxt_recv_msg_cmsg_pid(msg); + + rt = task->thread->runtime; + + pprocess = nxt_runtime_process_find(rt, sender); + + if (pprocess == NULL) { + /* The prototype may have exited before this was read. */ + nxt_debug(task, "REMOVE_CHILD_PID from unknown process %PI", sender); + goto done; + } + + if (nxt_slow_path(nxt_process_type(pprocess) != NXT_PROCESS_PROTOTYPE)) { + nxt_alert(task, "process %PI is not a prototype and cannot report " + "a child pid", sender); + goto done; + } + + nxt_queue_each(child, &pprocess->children, nxt_process_t, link) { + + if (child->parent_ns_pid != pid) { + continue; + } + + nxt_debug(task, "remove child pid %PI (aka %PI) of %PI", pid, + child->pid, sender); + + /* As in nxt_main_process_sigchld_handler(). */ + + if (!nxt_exiting) { + nxt_port_remove_notify_others(task, child); + } + + nxt_process_unlink(child); + + nxt_process_close_ports(task, child); + + goto done; + + } nxt_queue_loop; + + /* Not an error: a worker that died before WHOAMI left no record. */ + + nxt_debug(task, "process %PI reported child pid %PI, which it has no " + "record for", sender, pid); + +done: + + nxt_port_recv_msg_close_fds(msg); +} + +#endif + + static nxt_int_t nxt_main_process_port_create(nxt_task_t *task, nxt_runtime_t *rt) { @@ -1307,8 +1493,7 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) nxt_process_close_ports(task, process); nxt_queue_each(child, &children, nxt_process_t, link) { - nxt_queue_remove(&child->link); - child->link.next = NULL; + nxt_process_unlink(child); nxt_process_close_ports(task, child); } nxt_queue_loop; @@ -1327,8 +1512,7 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) nxt_queue_each(child, &children, nxt_process_t, link) { nxt_port_remove_notify_others(task, child); - nxt_queue_remove(&child->link); - child->link.next = NULL; + nxt_process_unlink(child); nxt_process_close_ports(task, child); } nxt_queue_loop; diff --git a/src/nxt_main_process.h b/src/nxt_main_process.h index 88fcf0b75..800ccc2ec 100644 --- a/src/nxt_main_process.h +++ b/src/nxt_main_process.h @@ -41,6 +41,12 @@ nxt_int_t nxt_main_test_run_file_store(nxt_task_t *task, const char *dir, const char *tmp_name, const char *name, u_char *buf, size_t size); void nxt_main_test_run_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +#if (NXT_USE_CMSG_PID) +void nxt_main_test_run_name_child(nxt_task_t *task, nxt_process_t *pprocess, + nxt_process_t *process, nxt_pid_t ns_pid); +void nxt_main_test_run_remove_child_pid_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +#endif #endif diff --git a/src/nxt_port.h b/src/nxt_port.h index 630102a31..5b3763abf 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -71,6 +71,12 @@ struct nxt_port_handlers_s { * inserting or reordering a slot renumbers the wire protocol. */ nxt_port_handler_t detached; + + /* + * A prototype reports a child that died before PROCESS_CREATED, by its + * namespace-local pid. Appended for the same reason as the slot above. + */ + nxt_port_handler_t remove_child_pid; }; @@ -130,6 +136,9 @@ typedef enum { _NXT_PORT_MSG_DETACHED = nxt_port_handler_idx(detached), + _NXT_PORT_MSG_REMOVE_CHILD_PID + = nxt_port_handler_idx(remove_child_pid), + NXT_PORT_MSG_MAX = sizeof(nxt_port_handlers_t) / sizeof(nxt_port_handler_t), @@ -175,9 +184,21 @@ typedef enum { NXT_PORT_MSG_READ_QUEUE = _NXT_PORT_MSG_READ_QUEUE, NXT_PORT_MSG_READ_SOCKET = _NXT_PORT_MSG_READ_SOCKET, NXT_PORT_MSG_DETACHED = nxt_msg_last(_NXT_PORT_MSG_DETACHED), + NXT_PORT_MSG_REMOVE_CHILD_PID + = nxt_msg_last(_NXT_PORT_MSG_REMOVE_CHILD_PID), } nxt_port_msg_type_t; +/* + * The message numbers are on the wire, and a language module built from + * another release reads the same numbers. Pin the last slot, so that a slot + * inserted before it fails the build. Add new slots after it and move the + * pin to the new last slot. + */ +nxt_static_assert(_NXT_PORT_MSG_REMOVE_CHILD_PID == 35, + "a port message slot was inserted, not appended"); + + /* * Wire-format payload for NXT_PORT_MSG_QUIT. A single byte selects * between fast and graceful exit on the receiving side. When the diff --git a/src/nxt_process.h b/src/nxt_process.h index 102cd6079..0ed9f371b 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -141,6 +141,15 @@ struct nxt_process_s { nxt_pid_t isolated_pid; + + /* + * The pid of this process inside the pid namespace of the prototype + * that forked it. Only main sets it, from the WHOAMI header; 0 means + * "no name". ->pid stays the global key. See REMOVE_CHILD_PID in + * src/nxt_main_process.c. + */ + nxt_pid_t parent_ns_pid; + const char *name; nxt_port_t *parent_port; @@ -225,6 +234,20 @@ NXT_EXPORT void nxt_process_arguments(nxt_task_t *task, char **orig_argv, #define nxt_process_port_first(process) \ nxt_queue_link_data(nxt_queue_first(&process->ports), nxt_port_t, link) +/* + * Take a process off its parent's "children" queue. link.next == NULL means + * "not linked", which the release build of nxt_queue_remove() does not set, + * so it is set here. Safe to call twice. + */ +nxt_inline void +nxt_process_unlink(nxt_process_t *process) +{ + if (process->link.next != NULL) { + nxt_queue_remove(&process->link); + process->link.next = NULL; + } +} + NXT_EXPORT void nxt_process_port_add(nxt_task_t *task, nxt_process_t *process, nxt_port_t *port); diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 0c38247a1..57e35b414 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -1610,13 +1610,10 @@ nxt_runtime_process_free(nxt_runtime_t *rt, nxt_process_t *process) { nxt_process_t *child; - if (process->link.next != NULL) { - nxt_queue_remove(&process->link); - } + nxt_process_unlink(process); nxt_queue_each(child, &process->children, nxt_process_t, link) { - nxt_queue_remove(&child->link); - child->link.next = NULL; + nxt_process_unlink(child); } nxt_queue_loop; nxt_assert(process->use_count == 0); diff --git a/src/test/nxt_main_remove_child_pid_test.c b/src/test/nxt_main_remove_child_pid_test.c new file mode 100644 index 000000000..7c1eddd96 --- /dev/null +++ b/src/test/nxt_main_remove_child_pid_test.c @@ -0,0 +1,463 @@ +/* + * Copyright (C) FreeUnit Community + */ + +/* + * Issue #310: under "isolation": {"namespaces": {"pid": true}} a worker that + * died before PROCESS_CREATED left its record, port and descriptor in main + * until the prototype exited. The prototype knows such a worker only by its + * namespace-local pid, which REMOVE_PID must not broadcast. + * + * Main keeps that pid next to the global one at WHOAMI time + * (nxt_main_process_name_child()) and resolves REMOVE_CHILD_PID against it, + * among the sender's own children only. Both halves are driven here. + * + * Naming must refuse 0, a negative pid and a name a live sibling holds, and + * must keep a name equal to the global pid. Resolving must refuse a short + * payload, pid 0, a negative pid, an unknown sender, a sender that is not a + * prototype and a pid no child of the sender holds; when it accepts, it must + * retire the record and tell the router by the global pid. + * + * Without NXT_USE_CMSG_PID the handler is not registered, and nothing is + * tested. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#if (NXT_USE_CMSG_PID) + +/* Namespace-local pids of two workers. */ +#define NXT_MAIN_RCP_NS_PID 7 +#define NXT_MAIN_RCP_OTHER_NS_PID 8 + + +typedef struct { + nxt_pid_t sender; + nxt_pid_t pid; + nxt_bool_t payload; + const char *label; +} nxt_main_rcp_case_t; + + +static nxt_port_t * +nxt_main_rcp_port(nxt_task_t *task, nxt_runtime_t *rt, nxt_pid_t pid, + nxt_process_type_t type) +{ + nxt_port_t *port; + + port = nxt_runtime_process_port_create(task, rt, pid, 0, type); + if (port != NULL) { + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + } + + return port; +} + + +static void +nxt_main_rcp_port_release(nxt_task_t *task, nxt_port_t *port) +{ + if (port != NULL) { + nxt_port_close(task, port); + nxt_runtime_port_remove(task, port); + } +} + + +/* + * Drive the handler once. The fixture ports are never write_ready, so what + * the handler sends stays on port->messages for the test to read. + */ + +static nxt_int_t +nxt_main_rcp_report(nxt_thread_t *thr, nxt_task_t *task, nxt_mp_t *mp, + nxt_pid_t sender, nxt_pid_t pid, nxt_bool_t payload) +{ + nxt_buf_t *buf; + nxt_port_recv_msg_t msg; + + buf = NULL; + + if (payload) { + buf = nxt_buf_mem_alloc(mp, sizeof(nxt_pid_t), 0); + if (nxt_slow_path(buf == NULL)) { + nxt_log_alert(thr->log, "main remove child pid test: no buffer"); + return NXT_ERROR; + } + + buf->mem.free = nxt_cpymem(buf->mem.free, &pid, sizeof(nxt_pid_t)); + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.buf = buf; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.type = _NXT_PORT_MSG_REMOVE_CHILD_PID; + msg.port_msg.pid = sender; + msg.cmsg_pid = sender; + + nxt_main_test_run_remove_child_pid_handler(task, &msg); + + if (buf != NULL) { + nxt_mp_free(mp, buf); + } + + return NXT_OK; +} + + +/* The router got one REMOVE_PID naming pid, and nothing else. */ + +static nxt_int_t +nxt_main_rcp_router_told(nxt_thread_t *thr, nxt_port_t *router_port, + nxt_pid_t pid, const char *label) +{ + nxt_pid_t sent_pid; + nxt_port_send_msg_t *sent; + + if (nxt_slow_path(nxt_queue_is_empty(&router_port->messages))) { + nxt_log_alert(thr->log, "main remove child pid test: %s: the router " + "was not told", label); + return NXT_ERROR; + } + + sent = nxt_queue_link_data(nxt_queue_first(&router_port->messages), + nxt_port_send_msg_t, link); + + if (nxt_slow_path(sent->port_msg.type != _NXT_PORT_MSG_REMOVE_PID + || sent->buf == NULL + || nxt_buf_used_size(sent->buf) != sizeof(nxt_pid_t))) + { + nxt_log_alert(thr->log, "main remove child pid test: %s: the router " + "got a message of type %d, not a REMOVE_PID", label, + (int) sent->port_msg.type); + return NXT_ERROR; + } + + nxt_memcpy(&sent_pid, sent->buf->mem.pos, sizeof(nxt_pid_t)); + + if (nxt_slow_path(sent_pid != pid)) { + nxt_log_alert(thr->log, "main remove child pid test: %s: the router " + "was told to remove pid %PI, not %PI", label, sent_pid, + pid); + return NXT_ERROR; + } + + if (nxt_slow_path(nxt_queue_next(nxt_queue_first(&router_port->messages)) + != nxt_queue_tail(&router_port->messages))) + { + nxt_log_alert(thr->log, "main remove child pid test: %s: the router " + "got more than one message", label); + return NXT_ERROR; + } + + /* Queued messages hold a port reference each; give them back. */ + nxt_port_test_run_error_handler(thr->task, router_port); + + return NXT_OK; +} + + +nxt_int_t +nxt_main_remove_child_pid_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_pid_t proto_pid, child_pid, other_pid, twin_pid; + nxt_task_t *task; + nxt_uint_t i; + nxt_port_t *router_port, *proto_port, *child_port, *other_port; + nxt_port_t *twin_port; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *proto, *child, *other, *twin; + nxt_event_engine_t engine, *saved_engine; + nxt_port_send_msg_t *sent; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "main remove child pid test started"); + + ret = NXT_ERROR; + router_port = NULL; + proto_port = NULL; + child_port = NULL; + other_port = NULL; + twin_port = NULL; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + /* nxt_port_remove_notify_others() allocates from engine->mem_pool. */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* + * This process stands in for main, so every fixture pid must differ from + * nxt_pid: nxt_port_remove_notify_others() skips the running process. + * twin is the worker whose local pid equals its global pid. + */ + + proto_pid = nxt_pid + 1; + child_pid = nxt_pid + 2; + other_pid = nxt_pid + 3; + twin_pid = nxt_pid + 5; + + router_port = nxt_main_rcp_port(task, rt, nxt_pid + 4, NXT_PROCESS_ROUTER); + proto_port = nxt_main_rcp_port(task, rt, proto_pid, NXT_PROCESS_PROTOTYPE); + child_port = nxt_main_rcp_port(task, rt, child_pid, NXT_PROCESS_APP); + other_port = nxt_main_rcp_port(task, rt, other_pid, NXT_PROCESS_APP); + twin_port = nxt_main_rcp_port(task, rt, twin_pid, NXT_PROCESS_APP); + + if (nxt_slow_path(router_port == NULL || proto_port == NULL + || child_port == NULL || other_port == NULL + || twin_port == NULL)) + { + goto done; + } + + proto = proto_port->process; + child = child_port->process; + other = other_port->process; + twin = twin_port->process; + + /* As nxt_main_process_whoami_handler() leaves the prototype's workers. */ + + nxt_queue_insert_tail(&proto->children, &child->link); + nxt_queue_insert_tail(&proto->children, &other->link); + nxt_queue_insert_tail(&proto->children, &twin->link); + + /* Naming: 0 and a negative pid are refused. */ + + nxt_main_test_run_name_child(task, proto, child, 0); + nxt_main_test_run_name_child(task, proto, child, -1); + + if (nxt_slow_path(child->parent_ns_pid != 0)) { + nxt_log_alert(thr->log, "main remove child pid test: a worker was " + "named %PI", child->parent_ns_pid); + goto done; + } + + nxt_main_test_run_name_child(task, proto, child, NXT_MAIN_RCP_NS_PID); + nxt_main_test_run_name_child(task, proto, other, + NXT_MAIN_RCP_OTHER_NS_PID); + + if (nxt_slow_path(child->parent_ns_pid != NXT_MAIN_RCP_NS_PID + || other->parent_ns_pid != NXT_MAIN_RCP_OTHER_NS_PID)) + { + nxt_log_alert(thr->log, "main remove child pid test: a worker was " + "not named"); + goto done; + } + + /* A name a live sibling holds is refused; the sibling keeps it. */ + + nxt_main_test_run_name_child(task, proto, twin, NXT_MAIN_RCP_NS_PID); + + if (nxt_slow_path(twin->parent_ns_pid != 0 + || child->parent_ns_pid != NXT_MAIN_RCP_NS_PID)) + { + nxt_log_alert(thr->log, "main remove child pid test: a worker was " + "allowed to claim a live sibling's name"); + goto done; + } + + /* + * Reports that must retire nothing and tell the router nothing. twin + * still has no name, so a handler that reads 0 as a name would retire + * it. The router holds a named child of its own for the non-prototype + * case, so that only the type check keeps that record. + */ + + nxt_queue_remove(&other->link); + nxt_queue_insert_tail(&router_port->process->children, &other->link); + + { + const nxt_main_rcp_case_t refused[] = { + { proto_pid, NXT_MAIN_RCP_NS_PID, 0, "a payload with no pid" }, + { proto_pid, 0, 1, "pid 0" }, + { proto_pid, -1, 1, "a negative pid" }, + { nxt_pid + 9, NXT_MAIN_RCP_NS_PID, 1, "an unknown sender" }, + { router_port->pid, NXT_MAIN_RCP_OTHER_NS_PID, 1, + "a sender that is not a prototype" }, + { proto_pid, NXT_MAIN_RCP_NS_PID + 100, 1, "a pid of no child" }, + }; + + for (i = 0; i < nxt_nitems(refused); i++) { + ret = nxt_main_rcp_report(thr, task, mp, refused[i].sender, + refused[i].pid, refused[i].payload); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = NXT_ERROR; + + if (nxt_slow_path(nxt_runtime_process_find(rt, child_pid) != child + || nxt_runtime_process_find(rt, other_pid) + != other + || nxt_runtime_process_find(rt, twin_pid) + != twin)) + { + nxt_log_alert(thr->log, "main remove child pid test: %s " + "retired a record", refused[i].label); + goto done; + } + + if (nxt_slow_path(!nxt_queue_is_empty(&router_port->messages))) { + sent = nxt_queue_link_data( + nxt_queue_first(&router_port->messages), + nxt_port_send_msg_t, link); + + nxt_log_alert(thr->log, "main remove child pid test: %s made " + "the router a message of type %d", + refused[i].label, (int) sent->port_msg.type); + goto done; + } + } + } + + nxt_queue_remove(&other->link); + nxt_queue_insert_tail(&proto->children, &other->link); + + /* A name equal to the global pid is kept. */ + + nxt_main_test_run_name_child(task, proto, twin, twin_pid); + + if (nxt_slow_path(twin->parent_ns_pid != twin_pid)) { + nxt_log_alert(thr->log, "main remove child pid test: a worker whose " + "two pids are equal was left unnamed"); + goto done; + } + + /* Accepted: the record goes and the router is told by the global pid. */ + + ret = nxt_main_rcp_report(thr, task, mp, proto_pid, twin_pid, 1); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = NXT_ERROR; + twin_port = NULL; + + if (nxt_slow_path(nxt_runtime_process_find(rt, twin_pid) != NULL)) { + nxt_log_alert(thr->log, "main remove child pid test: a worker whose " + "two pids are equal was not retired"); + goto done; + } + + if (nxt_slow_path(nxt_main_rcp_router_told(thr, router_port, twin_pid, + "equal pids") != NXT_OK)) + { + goto done; + } + + ret = nxt_main_rcp_report(thr, task, mp, proto_pid, NXT_MAIN_RCP_NS_PID, + 1); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + ret = NXT_ERROR; + child_port = NULL; + + if (nxt_slow_path(nxt_runtime_process_find(rt, child_pid) != NULL)) { + nxt_log_alert(thr->log, "main remove child pid test: the record of a " + "reported worker outlived the report"); + goto done; + } + + if (nxt_slow_path(nxt_runtime_process_find(rt, other_pid) != other)) { + nxt_log_alert(thr->log, "main remove child pid test: the report took " + "a sibling's record with it"); + goto done; + } + + if (nxt_slow_path(nxt_main_rcp_router_told(thr, router_port, child_pid, + "a reported worker") != NXT_OK)) + { + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "main remove child pid test passed"); + + ret = NXT_OK; + +done: + + /* + * The ports carry pools of their own; nxt_port_close() is a no-op on the + * -1 descriptors and nxt_runtime_port_remove() drops the last reference. + * nxt_runtime_process_free() unlinks a child still in a children queue. + */ + + if (router_port != NULL) { + nxt_port_test_run_error_handler(task, router_port); + } + + nxt_main_rcp_port_release(task, router_port); + nxt_main_rcp_port_release(task, child_port); + nxt_main_rcp_port_release(task, twin_port); + nxt_main_rcp_port_release(task, other_port); + nxt_main_rcp_port_release(task, proto_port); + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + + nxt_thread_mutex_destroy(&rt->processes_mutex); + + nxt_mp_destroy(mp); + + return ret; +} + +#else + +nxt_int_t +nxt_main_remove_child_pid_test(nxt_thread_t *thr) +{ + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main remove child pid test " + "skipped: no sender credentials on this platform"); + + return NXT_OK; +} + +#endif diff --git a/src/test/nxt_proto_creating_wedge_test.c b/src/test/nxt_proto_creating_wedge_test.c index e711c9dd8..490262b02 100644 --- a/src/test/nxt_proto_creating_wedge_test.c +++ b/src/test/nxt_proto_creating_wedge_test.c @@ -42,9 +42,10 @@ * That REMOVE_PID may only be sent when the pid in it is a usable global key, * which is what rt->is_pid_isolated says -- the same test nxt_process_create() * uses to decide whether a forked child may enter the runtime hash at all. - * The pid-isolated case is exercised here too, and it asserts the opposite: - * nothing at all must reach main, because sending a namespace-local pid would - * ask main to remove whatever unrelated process holds that number. + * The pid-isolated case is exercised here too: the namespace-local pid must + * not be broadcast, because it names an unrelated process everywhere else, so + * it goes to main alone as REMOVE_CHILD_PID. The router must still see + * nothing but the RPC error. * * The answer itself can fail to be written -- a full port queue or a failed * allocation, neither of which leaves anything for the initiator -- and then @@ -125,7 +126,7 @@ nxt_proto_wedge_expect(nxt_thread_t *thr, nxt_port_t *port, || nxt_buf_used_size(msg->buf) != sizeof(nxt_pid_t)) { nxt_log_alert(thr->log, "proto creating wedge test: %s: the %s " - "got a REMOVE_PID carrying no pid", label, whom); + "got a message carrying no pid", label, whom); return NXT_ERROR; } @@ -349,11 +350,9 @@ nxt_proto_creating_wedge_test(nxt_thread_t *thr) /* * The same child under pid isolation. ->pid is then the namespace-local - * pid fork() returned, and main keyed its record on the global pid it - * read from SCM_CREDENTIALS, so there is no pid to send: the initiator is - * still answered, and nothing at all may reach main. Sending the - * namespace-local pid would ask main to remove whatever unrelated process - * happens to hold that number, which is worse than the leak this closes. + * pid fork() returned, which REMOVE_PID must not broadcast. Main alone + * is told, with REMOVE_CHILD_PID; the router must see the RPC error and + * nothing else. */ rt->is_pid_isolated = 1; @@ -374,7 +373,11 @@ nxt_proto_creating_wedge_test(nxt_thread_t *thr) goto done; } - ret = nxt_proto_wedge_expect(thr, main_port, expect, 0, + expect[0].type = _NXT_PORT_MSG_REMOVE_CHILD_PID; + expect[0].stream = 0; + expect[0].pid = process->isolated_pid; + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 1, "a pid-isolated child that died while " "creating", "main process"); if (nxt_slow_path(ret != NXT_OK)) { @@ -437,11 +440,10 @@ nxt_proto_creating_wedge_test(nxt_thread_t *thr) * namespace-local one. * * What must not happen is a repair that looks like one: broadcasting that - * pid anyway would ask main and the router to remove whatever unrelated - * process holds the number, and a prototype's namespace-local counter - * climbs straight into the range the daemon's own pids occupy. So the - * assertion is an absence on both ports, and it is what turns red if a - * later change decides to send something here. + * pid anyway would ask the router to remove whatever unrelated process + * holds the number. So the router must stay silent here, and main gets + * REMOVE_CHILD_PID -- the injected failure is spent on the RPC error, so + * the report itself is written. * * The start RPC is left armed, which the code says out loud. It is * retired when the prototype itself dies -- the router keys that RPC on @@ -469,7 +471,11 @@ nxt_proto_creating_wedge_test(nxt_thread_t *thr) goto done; } - ret = nxt_proto_wedge_expect(thr, main_port, expect, 0, + expect[0].type = _NXT_PORT_MSG_REMOVE_CHILD_PID; + expect[0].stream = 0; + expect[0].pid = process->isolated_pid; + + ret = nxt_proto_wedge_expect(thr, main_port, expect, 1, "a pid-isolated child whose start error " "could not be written", "main process"); if (nxt_slow_path(ret != NXT_OK)) { diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index be33697d8..896d97915 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -278,6 +278,10 @@ main(int argc, char **argv) return 1; } + if (nxt_main_remove_child_pid_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_change_file_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 42baada4b..09ae15234 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -91,6 +91,7 @@ nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_main_whoami_test(nxt_thread_t *thr); +nxt_int_t nxt_main_remove_child_pid_test(nxt_thread_t *thr); nxt_int_t nxt_port_change_file_test(nxt_thread_t *thr); nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); From f72f93d081f8d04da2415ee39f453da7309fa838 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 21:36:11 +0000 Subject: [PATCH 287/391] cert: replace a certificate in place through the control API An external ACME client, such as certbot or lego, renews a certificate and must install the new bundle without downtime. Before this change, "PUT /certificates/" on an existing name returned 400. The old store was also not atomic: main opened "certs/" read-write without truncation, the controller wrote into it and ignored the result, and the metadata was saved before the file was written. Now a PUT on an existing name replaces the bundle in place. Controller: - It checks the name, the size (1 MiB or less, otherwise 413) and the bundle, and waits while a reconfiguration is in progress. - It publishes the new metadata first. This is the only step that can fail for lack of memory. Then it sends the bundle to main in a shared memory segment with the new port message CERT_STORE. The slot is appended to nxt_port_handlers_t, so no wire value changes, and the static pin moves to it. - When the store fails, it puts the old metadata back or removes the new name. This only frees memory, so it cannot fail. Other changes, "GET /certificates" and "GET /" wait until main answers. - When the current configuration names the bundle, it sends the same configuration to the router again. The router reads every bundle during a reconfiguration. Accepted connections keep their old TLS context. Main: - It accepts CERT_STORE only from the controller, checks the message layout, the name (not empty, no "/", no leading ".") and the size. - It copies the bundle with pread() and does not map the segment, so a sender that shrinks the segment cannot cause SIGBUS in main. - nxt_main_file_store() writes "certs/.store.tmp", syncs it, renames it over the bundle and syncs the directory. The temporary name is fixed, so a name of NAME_MAX bytes works. It is now a public function, and the test wrapper is removed. Router: CERT_GET opens the file read-only and never creates it. Responses: 200 "Certificate chain uploaded." for a new bundle or one that no listener names; 200 "Certificate chain updated." when the bundle is replaced and applied; 400 for a bad bundle, a wrong key or a dot name; 413 for a bundle over 1 MiB; 500 "Failed to store certificate." when main could not write the file (nothing changes); 500 "Certificate stored but not applied." when the router refused the configuration (the old certificate stays in use until the next reconfiguration or restart). Tests: test_tls_certificate_update is no longer skipped. New tests cover an unused bundle, a keep-alive connection, an in-flight request, a key mismatch, SNI with two bundles, a failed store, a name of 255 bytes, a restart and the 413 answer. The C test replaces a long bundle with a short one and checks for no tail and no temporary file. The restart test uses the requires_restart fixture and runs in the CI step that passes --restart. Docs: docs/acme.md (API behaviour, HTTP-01 route, deploy hook, control socket of each package, certbot and lego procedures, a systemd timer), OpenAPI, CHANGES, docs/changes.xml. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 10 +- CHANGES | 10 + docs/acme.md | 202 +++++++++++++++++ docs/changes.xml | 13 ++ docs/unit-openapi.yaml | 38 +++- src/nxt_cert.c | 325 ++++++++++++++++++++++++++-- src/nxt_cert.h | 13 +- src/nxt_controller.c | 202 ++++++++++++++--- src/nxt_main_process.c | 19 +- src/nxt_main_process.h | 4 +- src/nxt_port.h | 8 +- src/test/nxt_main_file_store_test.c | 43 +++- test/test_tls.py | 265 ++++++++++++++++++++++- 13 files changed, 1060 insertions(+), 92 deletions(-) create mode 100644 docs/acme.md diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ad7c59841..6c3c4c4de 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -826,13 +826,15 @@ jobs: if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} # test_asgi_lifespan.py stops unitd from inside the test to check the - # shutdown event. That needs --restart, which the run above does not - # pass, so both lifespan tests skip there. Run the file again with it. - - name: Run ASGI lifespan tests with --restart + # shutdown event, and test_tls_certificate_update_restart restarts it + # to check that a replaced bundle survives. Both need --restart, which + # the run above does not pass, so they skip there. Run them again. + - name: Run the tests that restart unitd, with --restart run: | export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" sudo -E /usr/bin/pytest-3 --print-log --restart \ - test/test_asgi_lifespan.py + test/test_asgi_lifespan.py \ + test/test_tls.py::test_tls_certificate_update_restart if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} # C unit-test suite (src/test/ -> build/tests): HTTP parser, port/IPC diff --git a/CHANGES b/CHANGES index 14fd34bd8..7be950b51 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Feature: "PUT /certificates/" replaces an existing certificate + bundle in place. Main writes the bundle to a temporary file and + renames it over the old one, so a crash never leaves a partial + bundle. When the current configuration names the bundle, the + controller applies the configuration again before it answers. New + handshakes get the new certificate; accepted connections finish + with the old one. An external ACME client can renew a certificate + with one request and no restart; see docs/acme.md. A bundle over + 1 MiB is refused with 413. The router opens bundles read-only. + *) Bugfix: with "isolation": {"namespaces": {"pid": true}}, a worker that died before it was fully started left its record, port and descriptor in the main process until the prototype exited. The diff --git a/docs/acme.md b/docs/acme.md new file mode 100644 index 000000000..ba02a8b65 --- /dev/null +++ b/docs/acme.md @@ -0,0 +1,202 @@ +# ACME certificates with an external client + +ACME is the protocol that certificate authorities use to issue and renew +certificates. Let's Encrypt is one of these authorities. FreeUnit does not +connect to a certificate authority. An external ACME client gets and renews +the certificate. Two examples are certbot and lego. A deploy hook then +uploads the new bundle through the control API. A bundle is one PEM body +that holds the server certificate, its chain, and the private key. + +Renewal needs no restart. `PUT /certificates/` on an existing name +replaces the bundle in place. When a listener names the bundle, FreeUnit +applies the configuration again. Connections that were already accepted +finish with the old certificate. New handshakes get the new certificate. + +This page gives the procedure for operators. + +## What the control API does + +- `PUT /certificates/` creates the bundle, or replaces it when the + name exists. The body is a PEM bundle. The first certificate must be the + server certificate. The chain certificates and the private key follow. +- Main writes the bundle to a temporary file in the certificate store and + renames it over the old file. A crash never leaves a partial bundle. +- The key must belong to the first certificate. The answer to a bundle + with the wrong key is `400 Invalid certificate.`. The old bundle stays in + use. +- A name that starts with `.` is reserved. The answer to such a name is + `400 Invalid certificate name.`. +- The answer to a bundle over 1 MiB is `413 Certificate bundle is too + large.`. +- When the current configuration names the bundle, the answer is + `200 Certificate chain updated.`. The router has loaded the new bundle + at that point. Otherwise the answer is `200 Certificate chain uploaded.`. +- `500 Failed to store certificate.` means main could not write the file. + Nothing changed. +- `500 Certificate stored but not applied.` means the new bundle is on + disk, but the router refused the configuration with it. One cause is a + `conf_commands` option that does not accept the new key type. The router + keeps the old certificate in use. `GET /certificates/` already + shows the metadata of the new bundle. The next reconfiguration or restart + loads the new bundle. Upload a working bundle again, or correct the + configuration. +- The request waits while another configuration change is in progress, as + `PUT /config` does. + +## HTTP-01 route on port 80 + +HTTP-01 is an ACME challenge type. The authority gets a token file over +plain HTTP from `http:///.well-known/acme-challenge/`. The +ACME client writes the token file into a directory. FreeUnit serves that +directory with `share`. The router runs as the `--user` user. The router +must be able to read the token files. It must also be able to enter each +directory above them. + +```json +{ + "listeners": { + "*:80": { "pass": "routes/http" }, + "*:443": { "pass": "routes/app", "tls": { "certificate": "example.org" } } + }, + "routes": { + "http": [ + { + "match": { "uri": "/.well-known/acme-challenge/*" }, + "action": { + "share": "/var/lib/unit-acme$uri", + "chroot": "/var/lib/unit-acme/.well-known/acme-challenge/", + "fallback": { "return": 404 } + } + }, + { "action": { "return": 301, "location": "https://$host$request_uri" } } + ], + "app": [ { "action": { "pass": "applications/app" } } ] + } +} +``` + +Leave out the `*:443` listener until the first bundle exists. FreeUnit +refuses a configuration where a listener names a missing bundle. + +## The deploy hook + +The hook uploads the renewed bundle under the same name every time. It +needs only `curl` and write access to the control socket. + +```sh +#!/bin/sh +# Deploy hook for certbot. lego uses it through the wrapper below. +# certbot sets RENEWED_LINEAGE, for example /etc/letsencrypt/live/example.org +# lego sets CERT_FULLCHAIN, CERT_KEY and UNIT_CERT_NAME through the wrapper +set -eu + +SOCK=${UNIT_CONTROL:-/var/run/control.freeunit.sock} +FULLCHAIN=${CERT_FULLCHAIN:-${RENEWED_LINEAGE:-}/fullchain.pem} +KEY=${CERT_KEY:-${RENEWED_LINEAGE:-}/privkey.pem} +NAME=${UNIT_CERT_NAME:-$(basename "${RENEWED_LINEAGE:?set RENEWED_LINEAGE or UNIT_CERT_NAME}")} + +cat "$FULLCHAIN" "$KEY" \ + | curl -fsS --unix-socket "$SOCK" -X PUT --data-binary @- \ + "http://localhost/certificates/$NAME" +``` + +Install it as `/usr/local/sbin/freeunit-deploy-hook`, mode 0755. + +The default socket is the socket of the FreeUnit Debian package. For a +different installation, set `UNIT_CONTROL` to its control socket: + +| Installation | Control socket | +|---|---| +| FreeUnit Debian package | `/var/run/control.freeunit.sock` | +| RPM package | `/var/run/unit/control.sock` | +| Docker image | `/var/run/control.unit.sock` | +| Source build | `/control.unit.sock`, or the `--control` option of `configure` | + +`unitd --help` shows the default socket of the binary under `--control`. +A service can set a different socket with `--control`. Look for this +option in the unit file of the service. + +`curl -f` returns an error for each answer of 400 or higher. The ACME +client then reports a failed upload. + +### certbot + +```sh +mkdir -p /etc/letsencrypt/renewal-hooks/deploy +ln -s /usr/local/sbin/freeunit-deploy-hook \ + /etc/letsencrypt/renewal-hooks/deploy/freeunit +certbot certonly --webroot -w /var/lib/unit-acme -d example.org -d www.example.org +RENEWED_LINEAGE=/etc/letsencrypt/live/example.org /usr/local/sbin/freeunit-deploy-hook +``` + +With the link, certbot runs the hook after each renewal. certbot sets +`RENEWED_LINEAGE` for the hook. The last line does the first upload. Add +the `*:443` listener after it. The `certbot.timer` unit of the distribution +runs `certbot renew` two times each day. + +### lego + +```sh +cat > /usr/local/sbin/lego-freeunit <<'EOF' +#!/bin/sh +CERT_FULLCHAIN=$LEGO_CERT_PATH CERT_KEY=$LEGO_CERT_KEY_PATH \ +UNIT_CERT_NAME=$LEGO_CERT_DOMAIN exec /usr/local/sbin/freeunit-deploy-hook +EOF +chmod 0755 /usr/local/sbin/lego-freeunit + +lego --email ops@example.org --accept-tos --path /var/lib/lego \ + --domains example.org --http --http.webroot /var/lib/unit-acme \ + run --run-hook /usr/local/sbin/lego-freeunit +``` + +lego sets `LEGO_CERT_PATH`, `LEGO_CERT_KEY_PATH` and `LEGO_CERT_DOMAIN` for +its hook. The wrapper maps them to the variables of the deploy hook. + +For a wildcard certificate, use `--dns ` instead of `--http`. +FreeUnit then needs no challenge route. + +### A renewal timer for lego + +lego has no timer. Use a systemd timer that runs one time each day. lego +renews a certificate only when it is close to expiry. + +```ini +# /etc/systemd/system/lego-renew.service +[Unit] +Description=Renew certificates with lego and deploy them to FreeUnit + +[Service] +Type=oneshot +ExecStart=/usr/bin/lego --email ops@example.org --accept-tos \ + --path /var/lib/lego --domains example.org \ + --http --http.webroot /var/lib/unit-acme \ + renew --no-random-sleep --renew-hook /usr/local/sbin/lego-freeunit +``` + +```ini +# /etc/systemd/system/lego-renew.timer +[Unit] +Description=Daily certificate renewal + +[Timer] +OnCalendar=daily +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target +``` + +```sh +systemctl enable --now lego-renew.timer +``` + +To use cron instead, add this line: `17 3 * * * root /usr/bin/lego ... renew --no-random-sleep --renew-hook /usr/local/sbin/lego-freeunit`. + +## Notes + +- Do not manually edit the files in the state directory. The next + reconfiguration loads a bundle that you changed there, but + `GET /certificates` shows the old metadata until a restart. +- The hook must run as a user that can write to the control socket. Do + not give an application process access to the socket. diff --git a/docs/changes.xml b/docs/changes.xml index 1ed99f059..9ea5d87d6 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,19 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +"PUT /certificates/<name>" replaces an existing certificate bundle in +place. Main writes the bundle to a temporary file and renames it over the +old one, so a crash never leaves a partial bundle. When the current +configuration names the bundle, the controller applies the configuration +again before it answers. New handshakes get the new certificate; accepted +connections finish with the old one. An external ACME client can renew a +certificate with one request and no restart; see docs/acme.md. A bundle over +1 MiB is refused with 413. The router opens bundles read-only. + + + with "isolation": {"namespaces": {"pid": true}}, a worker that died before it diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 4492593df..e52016501 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -112,14 +112,25 @@ paths: put: operationId: putCertBundle - summary: "Create or overwrite the actual certificate bundle" - description: "Creates or overwrites the [stored certificate bundle] - (https://unit.nginx.org/certificates/) in Unit. The body is a PEM - bundle: the server certificate first, then its chain, then the - private key. A bundle whose key does not belong to the first - certificate is refused with 400 `Invalid certificate.`; a name that - starts with `.` is reserved and refused with 400 - `Invalid certificate name.`." + summary: "Create or replace the actual certificate bundle" + description: "Creates the [stored certificate bundle] + (https://unit.nginx.org/certificates/) in Unit, or replaces it in + place when the name exists. The body is a PEM bundle: the server + certificate first, then its chain, then the private key. Main + writes the bundle to a temporary file and renames it over the old + one. When the current configuration names the bundle, the + controller applies the configuration again before it answers with + `Certificate chain updated.`; new handshakes get the new + certificate, accepted connections finish with the old one. + Otherwise the answer is `Certificate chain uploaded.`. A bundle + whose key does not belong to the first certificate is refused with + 400 `Invalid certificate.`, a name that starts with `.` with 400 + `Invalid certificate name.`, and a bundle over 1 MiB with 413 + `Certificate bundle is too large.`; the old bundle stays in use. + 500 `Certificate stored but not applied.` means the bundle is on + disk, but the router refused the configuration with it and keeps + the old certificate. The request waits while a reconfiguration is + in progress, as a configuration change does." tags: - certificates @@ -145,6 +156,9 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + "413": + $ref: "#/components/responses/responsePayloadTooLarge" + "500": $ref: "#/components/responses/responseInternalError" @@ -6553,6 +6567,14 @@ components: example1: $ref: "#/components/examples/errorValueDoesntExist" + responsePayloadTooLarge: + description: "Payload Too Large. The certificate bundle is over 1 MiB." + + content: + application/json: + schema: + $ref: "#/components/schemas/jsonErrorMessage" + responseInternalError: description: "Internal server error; the configuration wasn't applied. This may occur with misconfigured paths, wrong permissions, etc." diff --git a/src/nxt_cert.c b/src/nxt_cert.c index 9989fddad..cdb9159f3 100644 --- a/src/nxt_cert.c +++ b/src/nxt_cert.c @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -46,11 +47,11 @@ struct nxt_cert_s { }; -typedef struct { +struct nxt_cert_info_s { nxt_str_t name; nxt_conf_value_t *value; nxt_mp_t *mp; -} nxt_cert_info_t; +}; typedef struct { @@ -439,15 +440,39 @@ nxt_cert_info_init(nxt_task_t *task, nxt_array_t *certs) nxt_int_t nxt_cert_info_save(nxt_str_t *name, nxt_cert_t *cert) { - nxt_mp_t *mp; - nxt_int_t ret; - nxt_cert_info_t *info; - nxt_conf_value_t *value; - nxt_lvlhsh_query_t lhq; + nxt_cert_info_t *info, *old; + + info = nxt_cert_info_create(name, cert); + if (nxt_slow_path(info == NULL)) { + return NXT_ERROR; + } + + if (nxt_slow_path(nxt_cert_info_replace(info, &old) != NXT_OK)) { + nxt_cert_info_release(info); + return NXT_ERROR; + } + + nxt_cert_info_release(old); + + return NXT_OK; +} + + +/* + * Make the metadata of a bundle without publishing it. The controller calls + * this and nxt_cert_info_replace() before main replaces the file, so that a + * failed allocation does not change the stored bundle. + */ +nxt_cert_info_t * +nxt_cert_info_create(nxt_str_t *name, nxt_cert_t *cert) +{ + nxt_mp_t *mp; + nxt_cert_info_t *info; + nxt_conf_value_t *value; mp = nxt_mp_create(1024, 128, 256, 32); if (nxt_slow_path(mp == NULL)) { - return NXT_ERROR; + return NULL; } info = nxt_mp_get(mp, sizeof(nxt_cert_info_t)); @@ -468,28 +493,78 @@ nxt_cert_info_save(nxt_str_t *name, nxt_cert_t *cert) info->mp = mp; info->value = value; - lhq.key_hash = nxt_djb_hash(name->start, name->length); + return info; + +fail: + + nxt_mp_destroy(mp); + return NULL; +} + + +/* + * Put the metadata in the hash. "old" gets the old metadata of the same + * name, or NULL when the name is new. The caller keeps "old" until it calls + * nxt_cert_info_release() or nxt_cert_info_restore(). On failure, the hash + * does not change. + */ +nxt_int_t +nxt_cert_info_replace(nxt_cert_info_t *info, nxt_cert_info_t **old) +{ + nxt_int_t ret; + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(info->name.start, info->name.length); lhq.replace = 1; - lhq.key = *name; + lhq.key = info->name; lhq.value = info; lhq.proto = &nxt_cert_info_hash_proto; ret = nxt_lvlhsh_insert(&nxt_cert_info, &lhq); if (nxt_slow_path(ret != NXT_OK)) { - goto fail; + return NXT_ERROR; } - if (lhq.value != info) { - info = lhq.value; - nxt_mp_destroy(info->mp); - } + *old = (lhq.value != info) ? lhq.value : NULL; return NXT_OK; +} -fail: - nxt_mp_destroy(mp); - return NXT_ERROR; +/* + * Undo nxt_cert_info_replace(): put the old metadata back, or remove the + * name when it was new. Both only free memory, so this cannot fail. + * Releases "info". + */ +void +nxt_cert_info_restore(nxt_cert_info_t *info, nxt_cert_info_t *old) +{ + nxt_lvlhsh_query_t lhq; + + lhq.key_hash = nxt_djb_hash(info->name.start, info->name.length); + lhq.key = info->name; + lhq.proto = &nxt_cert_info_hash_proto; + + if (old != NULL) { + lhq.replace = 1; + lhq.value = old; + + (void) nxt_lvlhsh_insert(&nxt_cert_info, &lhq); + + } else { + (void) nxt_lvlhsh_delete(&nxt_cert_info, &lhq); + } + + nxt_cert_info_release(info); +} + + +void +nxt_cert_info_release(nxt_cert_info_t *info) +{ + if (info != NULL) { + nxt_mp_destroy(info->mp); + } } @@ -1274,7 +1349,7 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_slow_path(port->type != NXT_PROCESS_CONTROLLER && port->type != NXT_PROCESS_ROUTER)) { - nxt_alert(task, "process %PI cannot store certificates", + nxt_alert(task, "process %PI cannot read certificates", nxt_recv_msg_cmsg_pid(msg)); nxt_port_recv_msg_close_fds(msg); return; @@ -1304,7 +1379,7 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) p = nxt_cpymem(file.name, rt->certs.start, rt->certs.length); p = nxt_cpymem(p, name.start, name.length + 1); - ret = nxt_file_open(task, &file, NXT_FILE_RDWR, NXT_FILE_CREATE_OR_OPEN, + ret = nxt_file_open(task, &file, NXT_FILE_RDONLY, NXT_FILE_OPEN, NXT_FILE_OWNER_ACCESS); nxt_free(file.name); @@ -1334,6 +1409,216 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Send "mbuf" to main to store it as the bundle "name". The bundle goes in + * a shared memory segment, as conf.json does. The port message holds the + * NUL-terminated name, followed by the size. "handler" gets RPC_READY when + * the file is in place, and RPC_ERROR or a NULL message otherwise. + */ +void +nxt_cert_store_put(nxt_task_t *task, nxt_str_t *name, nxt_buf_mem_t *mbuf, + nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx) +{ + void *mem; + size_t size; + uint32_t stream; + nxt_fd_t fd; + nxt_int_t ret; + nxt_buf_t *b; + nxt_port_t *main_port, *ctl_port; + nxt_runtime_t *rt; + + size = nxt_buf_mem_used_size(mbuf); + + fd = nxt_shm_open(task, size); + if (nxt_slow_path(fd == -1)) { + goto fail; + } + + mem = nxt_mem_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + goto fail; + } + + nxt_memcpy(mem, mbuf->pos, size); + nxt_mem_munmap(mem, size); + + b = nxt_buf_mem_alloc(mp, name->length + 1 + sizeof(size_t), 0); + if (nxt_slow_path(b == NULL)) { + goto fail; + } + + b->completion_handler = nxt_cert_buf_completion; + + nxt_buf_cpystr(b, name); + *b->mem.free++ = '\0'; + b->mem.free = nxt_cpymem(b->mem.free, &size, sizeof(size_t)); + + rt = task->thread->runtime; + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; + + stream = nxt_port_rpc_register_handler(task, ctl_port, handler, handler, + -1, ctx); + if (nxt_slow_path(stream == 0)) { + goto fail; + } + + ret = nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_CERT_STORE | NXT_PORT_MSG_CLOSE_FD, + fd, stream, ctl_port->id, b); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_rpc_cancel(task, ctl_port, stream); + goto fail; + } + + /* See nxt_cert_store_get(). */ + nxt_mp_retain(mp); + + return; + +fail: + + /* The port layer owns the descriptor only after a successful write. */ + if (fd != -1) { + nxt_fd_close(fd); + } + + handler(task, NULL, ctx); +} + + +/* + * Main's side of nxt_cert_store_put(). nxt_main_file_store() writes the + * bundle to "certs/.store.tmp" and renames it over "certs/", so a + * reader sees the old bundle or the new one, never a partial file. The + * temporary name is fixed, so a name of NAME_MAX bytes still works, and one + * name is enough: the controller sends one bundle at a time, and a bundle + * name cannot start with ".". The controller is unprivileged, so main + * checks the sender and the name again. + */ +void +nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + size_t size, used; + u_char *p, *path, *mem; + nxt_str_t name; + nxt_file_t file; + nxt_port_t *port; + nxt_runtime_t *rt; + nxt_port_msg_type_t type; + + port = nxt_runtime_port_find(task->thread->runtime, + nxt_recv_msg_cmsg_pid(msg), + msg->port_msg.reply_port); + + if (nxt_slow_path(port == NULL || port->type != NXT_PROCESS_CONTROLLER)) { + nxt_alert(task, "process %PI cannot store certificates", + nxt_recv_msg_cmsg_pid(msg)); + nxt_port_recv_msg_close_fds(msg); + return; + } + + rt = task->thread->runtime; + + path = NULL; + mem = NULL; + type = NXT_PORT_MSG_RPC_ERROR; + + if (nxt_slow_path(rt->certs.start == NULL)) { + nxt_alert(task, "no certificates storage directory"); + goto done; + } + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_alert(task, "cert_store_handler: invalid shm fd"); + goto done; + } + + /* The message holds the NUL-terminated name, followed by the size. */ + + used = nxt_buf_mem_used_size(&msg->buf->mem); + + name.start = msg->buf->mem.pos; + p = memchr(name.start, '\0', used); + + if (nxt_slow_path(p == NULL + || (size_t) (p - name.start) + 1 + sizeof(size_t) != used + || p == name.start + || name.start[0] == '.' + || memchr(name.start, '/', p - name.start) != NULL)) + { + nxt_alert(task, "cert_store_handler: invalid certificate name"); + goto done; + } + + name.length = p - name.start; + + nxt_memcpy(&size, p + 1, sizeof(size_t)); + + if (nxt_slow_path(size == 0 || size > NXT_CERT_STORE_MAX_SIZE)) { + nxt_alert(task, "cert_store_handler: invalid bundle size %uz", size); + goto done; + } + + /* + * Copy the bundle with pread() instead of mapping the segment: a + * sender can shrink a mapped segment, and a read past its end is + * SIGBUS in main. A short pread() is only an error. + */ + + mem = nxt_malloc(size); + if (nxt_slow_path(mem == NULL)) { + goto done; + } + + nxt_memzero(&file, sizeof(nxt_file_t)); + file.fd = msg->fd[0]; + + /* nxt_file_read() logs the name when pread() fails; it must not be NULL. */ + file.name = name.start; + + if (nxt_slow_path(nxt_file_read(&file, mem, size, 0) != (ssize_t) size)) { + nxt_alert(task, "cert_store_handler: short bundle read"); + goto done; + } + + /* The buffer holds "/", a NUL, and "/.store.tmp". */ + + path = nxt_malloc(2 * rt->certs.length + name.length + 1 + + nxt_length(".store.tmp") + 1); + if (nxt_slow_path(path == NULL)) { + goto done; + } + + p = nxt_cpymem(path, rt->certs.start, rt->certs.length); + p = nxt_cpymem(p, name.start, name.length + 1); + p = nxt_cpymem(p, rt->certs.start, rt->certs.length); + nxt_memcpy(p, ".store.tmp", nxt_length(".store.tmp") + 1); + + if (nxt_main_file_store(task, (char *) rt->certs.start, + (char *) path + rt->certs.length + name.length + 1, + (char *) path, mem, size) + == NXT_OK) + { + type = NXT_PORT_MSG_RPC_READY_LAST; + + } else { + nxt_alert(task, "failed to store certificate \"%V\"", &name); + } + +done: + + (void) nxt_port_socket_write(task, port, type, -1, msg->port_msg.stream, + 0, NULL); + + nxt_free(mem); + nxt_free(path); + nxt_port_recv_msg_close_fds(msg); +} + + void nxt_cert_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) { diff --git a/src/nxt_cert.h b/src/nxt_cert.h index dbaddcf93..3f3ee9788 100644 --- a/src/nxt_cert.h +++ b/src/nxt_cert.h @@ -8,25 +8,36 @@ #define _NXT_CERT_INCLUDED_ -typedef struct nxt_cert_s nxt_cert_t; +typedef struct nxt_cert_s nxt_cert_t; +typedef struct nxt_cert_info_s nxt_cert_info_t; nxt_cert_t *nxt_cert_mem(nxt_task_t *task, nxt_buf_mem_t *mbuf); void nxt_cert_destroy(nxt_cert_t *cert); void nxt_cert_info_init(nxt_task_t *task, nxt_array_t *certs); nxt_int_t nxt_cert_info_save(nxt_str_t *name, nxt_cert_t *cert); +nxt_cert_info_t *nxt_cert_info_create(nxt_str_t *name, nxt_cert_t *cert); +nxt_int_t nxt_cert_info_replace(nxt_cert_info_t *info, nxt_cert_info_t **old); +void nxt_cert_info_restore(nxt_cert_info_t *info, nxt_cert_info_t *old); +void nxt_cert_info_release(nxt_cert_info_t *info); nxt_conf_value_t *nxt_cert_info_get(nxt_str_t *name); nxt_conf_value_t *nxt_cert_info_get_all(nxt_mp_t *mp); nxt_int_t nxt_cert_info_delete(nxt_str_t *name); +/* A stored bundle is a chain and a key. 1 MiB can hold hundreds. */ +#define NXT_CERT_STORE_MAX_SIZE (1024 * 1024) + nxt_array_t *nxt_cert_store_load(nxt_task_t *task, nxt_mp_t *mem_pool); void nxt_cert_store_release(nxt_array_t *certs); void nxt_cert_store_get(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx); +void nxt_cert_store_put(nxt_task_t *task, nxt_str_t *name, nxt_buf_mem_t *mbuf, + nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx); void nxt_cert_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp); void nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +void nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_cert_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); #endif /* _NXT_CERT_INCLUDED_ */ diff --git a/src/nxt_controller.c b/src/nxt_controller.c index e7c2bb0d9..c5688b6be 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -29,6 +29,16 @@ typedef struct { } nxt_controller_request_t; +#if (NXT_TLS) +typedef struct { + nxt_str_t name; + nxt_cert_info_t *info; + nxt_cert_info_t *old; + nxt_controller_request_t *req; +} nxt_controller_cert_store_t; +#endif + + typedef struct { nxt_uint_t status; nxt_conf_value_t *conf; @@ -98,7 +108,9 @@ static void nxt_controller_status_response(nxt_task_t *task, #if (NXT_TLS) static void nxt_controller_process_cert(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path); -static void nxt_controller_process_cert_save(nxt_task_t *task, +static void nxt_controller_cert_store_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg, void *data); +static void nxt_controller_cert_apply_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static nxt_bool_t nxt_controller_cert_in_use(nxt_str_t *name); static void nxt_controller_cert_cleanup(nxt_task_t *task, void *obj, @@ -139,6 +151,9 @@ static nxt_uint_t nxt_controller_router_ready; static nxt_controller_conf_t nxt_controller_conf; static nxt_queue_t nxt_controller_waiting_requests; static nxt_bool_t nxt_controller_waiting_init_conf; +#if (NXT_TLS) +static nxt_bool_t nxt_controller_cert_storing; +#endif static nxt_conf_value_t *nxt_controller_status; @@ -1328,6 +1343,15 @@ nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req) goto invalid_method; } +#if (NXT_TLS) + /* The answer has the certificates. Wait, as in the cert handler. */ + if (nxt_controller_cert_storing) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, + &req->link); + return; + } +#endif + if (nxt_controller_status == NULL) { nxt_controller_process_status(task, req); return; @@ -1866,13 +1890,13 @@ static void nxt_controller_process_cert(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path) { - u_char *p; - nxt_str_t name; - nxt_int_t ret; - nxt_conn_t *c; - nxt_cert_t *cert; - nxt_conf_value_t *value; - nxt_controller_response_t resp; + u_char *p; + nxt_str_t name; + nxt_conn_t *c; + nxt_cert_t *cert; + nxt_conf_value_t *value; + nxt_controller_response_t resp; + nxt_controller_cert_store_t *store; name.length = path->length - 1; name.start = path->start + 1; @@ -1895,6 +1919,13 @@ nxt_controller_process_cert(nxt_task_t *task, if (nxt_str_eq(&req->parser.method, "GET", 3)) { + /* While main stores a bundle, its metadata can roll back. Wait. */ + if (nxt_controller_cert_storing) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, + &req->link); + return; + } + if (name.length != 0) { value = nxt_cert_info_get(&name); if (value == NULL) { @@ -1927,10 +1958,19 @@ nxt_controller_process_cert(nxt_task_t *task, goto invalid_name; } + /* + * A store or a delete waits for the current reconfiguration: the router + * reads the bundles while it applies a configuration. + */ + if (nxt_controller_check_postpone_request(task)) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, &req->link); + return; + } + if (nxt_str_eq(&req->parser.method, "PUT", 3)) { - value = nxt_cert_info_get(&name); - if (value != NULL) { - goto exists_cert; + /* Main refuses a larger bundle. Send 413 here, not 500. */ + if (nxt_buf_mem_used_size(&c->read->mem) > NXT_CERT_STORE_MAX_SIZE) { + goto too_large; } cert = nxt_cert_mem(task, &c->read->mem); @@ -1938,16 +1978,42 @@ nxt_controller_process_cert(nxt_task_t *task, goto invalid_cert; } - ret = nxt_cert_info_save(&name, cert); + store = nxt_mp_get(c->mem_pool, sizeof(nxt_controller_cert_store_t)); + if (nxt_slow_path(store == NULL)) { + nxt_cert_destroy(cert); + goto alloc_fail; + } + + /* + * Publish the new metadata before main replaces the file. All the + * allocations are here, so a failure changes nothing on disk, and + * the rollback after a failed store cannot fail. + */ + store->info = nxt_cert_info_create(&name, cert); nxt_cert_destroy(cert); - if (nxt_slow_path(ret != NXT_OK)) { + if (nxt_slow_path(store->info == NULL)) { + goto alloc_fail; + } + + if (nxt_slow_path(nxt_cert_info_replace(store->info, &store->old) + != NXT_OK)) + { + nxt_cert_info_release(store->info); goto alloc_fail; } - nxt_cert_store_get(task, &name, c->mem_pool, - nxt_controller_process_cert_save, req); + nxt_controller_cert_storing = 1; + + store->name = name; + store->req = req; + + /* Any other change waits until main answers. */ + nxt_queue_insert_head(&nxt_controller_waiting_requests, &req->link); + + nxt_cert_store_put(task, &name, &c->read->mem, c->mem_pool, + nxt_controller_cert_store_handler, store); return; } @@ -1995,10 +2061,10 @@ nxt_controller_process_cert(nxt_task_t *task, nxt_controller_response(task, req, &resp); return; -exists_cert: +too_large: - resp.status = 400; - resp.title = (u_char *) "Certificate already exists."; + resp.status = 413; + resp.title = (u_char *) "Certificate bundle is too large."; resp.offset = -1; nxt_controller_response(task, req, &resp); @@ -2042,42 +2108,106 @@ nxt_controller_process_cert(nxt_task_t *task, } +/* + * Main stored the bundle, or the store failed. On failure, the old metadata + * goes back, so a failed store leaves no metadata without a file. When the + * current configuration names the bundle, the controller sends the + * configuration to the router again. The router reads every bundle during a + * reconfiguration, so new handshakes get the new certificate. Accepted + * connections keep their old TLS context. + */ static void -nxt_controller_process_cert_save(nxt_task_t *task, nxt_port_recv_msg_t *msg, +nxt_controller_cert_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_conn_t *c; - nxt_buf_mem_t *mbuf; - nxt_controller_request_t *req; - nxt_controller_response_t resp; + nxt_int_t rc; + nxt_runtime_t *rt; + nxt_controller_request_t *req; + nxt_controller_response_t resp; + nxt_controller_cert_store_t *store; - req = data; + store = data; + req = store->req; + + nxt_queue_remove(&req->link); nxt_memzero(&resp, sizeof(nxt_controller_response_t)); - if (msg == NULL || msg->port_msg.type == _NXT_PORT_MSG_RPC_ERROR) { + nxt_controller_cert_storing = 0; + + if (msg == NULL || msg->port_msg.type != NXT_PORT_MSG_RPC_READY) { + nxt_cert_info_restore(store->info, store->old); + resp.status = 500; resp.title = (u_char *) "Failed to store certificate."; - - nxt_controller_response(task, req, &resp); - return; + resp.offset = -1; + goto done; } - c = req->conn; + nxt_cert_info_release(store->old); - mbuf = &c->read->mem; + rt = task->thread->runtime; - nxt_fd_write(msg->fd[0], mbuf->pos, nxt_buf_mem_used_size(mbuf)); + /* When no router is ready, the next router loads the new bundle. */ - nxt_fd_close(msg->fd[0]); - msg->fd[0] = -1; + if (nxt_controller_cert_in_use(&store->name) + && nxt_controller_router_ready + && rt->port_by_type[NXT_PROCESS_ROUTER] != NULL) + { + rc = nxt_controller_conf_send(task, req->conn->mem_pool, + nxt_controller_conf.root, + nxt_controller_cert_apply_handler, req); - nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + if (nxt_fast_path(rc == NXT_OK)) { + nxt_queue_insert_head(&nxt_controller_waiting_requests, + &req->link); + return; + } + + resp.status = 500; + resp.title = (u_char *) "Certificate stored but not applied."; + resp.offset = -1; + goto done; + } resp.status = 200; resp.title = (u_char *) "Certificate chain uploaded."; +done: + nxt_controller_response(task, req, &resp); + + nxt_controller_flush_requests(task); +} + + +static void +nxt_controller_cert_apply_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, + void *data) +{ + nxt_controller_request_t *req; + nxt_controller_response_t resp; + + req = data; + + nxt_queue_remove(&req->link); + + nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + + if (msg->port_msg.type == NXT_PORT_MSG_RPC_READY) { + resp.status = 200; + resp.title = (u_char *) "Certificate chain updated."; + + } else { + /* The router kept the old configuration and the old contexts. */ + resp.status = 500; + resp.title = (u_char *) "Certificate stored but not applied."; + resp.offset = -1; + } + + nxt_controller_response(task, req, &resp); + + nxt_controller_flush_requests(task); } @@ -2731,6 +2861,10 @@ nxt_controller_response(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_set(&status_line, "405 Method Not Allowed"); break; + case 413: + nxt_str_set(&status_line, "413 Payload Too Large"); + break; + default: nxt_str_set(&status_line, "500 Internal Server Error"); break; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index ad84cf2cf..cda28c8e2 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -68,8 +68,6 @@ static void nxt_main_remove_child_pid_handler(nxt_task_t *task, #endif static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); -static nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *dir, - const char *tmp_name, const char *name, u_char *buf, size_t size); static nxt_int_t nxt_main_file_store_inherit(nxt_task_t *task, nxt_file_t *tmp, const char *name); static void nxt_main_port_access_log_handler(nxt_task_t *task, @@ -751,20 +749,6 @@ nxt_main_test_run_start_process_handler(nxt_task_t *task, } -/* - * Public wrapper that lets src/test/nxt_main_file_store_test.c drive the - * static nxt_main_file_store() against a scratch directory -- used to - * verify that the store is atomic and never damages the existing file - * (issue #215). - */ -nxt_int_t -nxt_main_test_run_file_store(nxt_task_t *task, const char *dir, - const char *tmp_name, const char *name, u_char *buf, size_t size) -{ - return nxt_main_file_store(task, dir, tmp_name, name, buf, size); -} - - /* Lets src/test/nxt_main_whoami_test.c drive the WHOAMI handler. */ void nxt_main_test_run_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) @@ -882,6 +866,7 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { #if (NXT_TLS) .cert_get = nxt_cert_store_get_handler, .cert_delete = nxt_cert_store_delete_handler, + .cert_store = nxt_cert_store_put_handler, #endif #if (NXT_HAVE_NJS) .script_get = nxt_script_store_get_handler, @@ -2216,7 +2201,7 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) * exists, its mode and ownership are carried over to the replacement, so a * state file an administrator has re-permissioned keeps its settings. */ -static nxt_int_t +nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *dir, const char *tmp_name, const char *name, u_char *buf, size_t size) { diff --git a/src/nxt_main_process.h b/src/nxt_main_process.h index 800ccc2ec..09ac6cf4a 100644 --- a/src/nxt_main_process.h +++ b/src/nxt_main_process.h @@ -21,6 +21,8 @@ typedef enum { nxt_int_t nxt_main_process_start(nxt_thread_t *thr, nxt_task_t *task, nxt_runtime_t *runtime); +nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *dir, + const char *tmp_name, const char *name, u_char *buf, size_t size); NXT_EXPORT extern nxt_uint_t nxt_conf_ver; @@ -37,8 +39,6 @@ extern const nxt_sig_event_t nxt_process_signals[]; void nxt_main_test_process_new_failures(nxt_uint_t failures); void nxt_main_test_run_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); -nxt_int_t nxt_main_test_run_file_store(nxt_task_t *task, const char *dir, - const char *tmp_name, const char *name, u_char *buf, size_t size); void nxt_main_test_run_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); #if (NXT_USE_CMSG_PID) diff --git a/src/nxt_port.h b/src/nxt_port.h index 5b3763abf..4e3f23a51 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -77,6 +77,9 @@ struct nxt_port_handlers_s { * namespace-local pid. Appended for the same reason as the slot above. */ nxt_port_handler_t remove_child_pid; + + /* The controller asks main to store a certificate bundle. Appended. */ + nxt_port_handler_t cert_store; }; @@ -139,6 +142,8 @@ typedef enum { _NXT_PORT_MSG_REMOVE_CHILD_PID = nxt_port_handler_idx(remove_child_pid), + _NXT_PORT_MSG_CERT_STORE = nxt_port_handler_idx(cert_store), + NXT_PORT_MSG_MAX = sizeof(nxt_port_handlers_t) / sizeof(nxt_port_handler_t), @@ -186,6 +191,7 @@ typedef enum { NXT_PORT_MSG_DETACHED = nxt_msg_last(_NXT_PORT_MSG_DETACHED), NXT_PORT_MSG_REMOVE_CHILD_PID = nxt_msg_last(_NXT_PORT_MSG_REMOVE_CHILD_PID), + NXT_PORT_MSG_CERT_STORE = nxt_msg_last(_NXT_PORT_MSG_CERT_STORE), } nxt_port_msg_type_t; @@ -195,7 +201,7 @@ typedef enum { * inserted before it fails the build. Add new slots after it and move the * pin to the new last slot. */ -nxt_static_assert(_NXT_PORT_MSG_REMOVE_CHILD_PID == 35, +nxt_static_assert(_NXT_PORT_MSG_CERT_STORE == 36, "a port message slot was inserted, not appended"); diff --git a/src/test/nxt_main_file_store_test.c b/src/test/nxt_main_file_store_test.c index fb32dcfb2..6c9e49af4 100644 --- a/src/test/nxt_main_file_store_test.c +++ b/src/test/nxt_main_file_store_test.c @@ -123,9 +123,8 @@ static nxt_int_t nxt_main_file_store_test_store(nxt_task_t *task, const char *dir, const char *tmp_name, const char *name, const char *content) { - return nxt_main_test_run_file_store(task, dir, tmp_name, name, - (u_char *) content, - nxt_strlen(content)); + return nxt_main_file_store(task, dir, tmp_name, name, (u_char *) content, + nxt_strlen(content)); } @@ -405,6 +404,44 @@ nxt_main_file_store_test(nxt_thread_t *thr) (void) unlink(victim); + /* + * Replace a certificate bundle under its own name. The temporary file + * is in the same directory and has a reserved dot name. A short bundle + * that replaces a long bundle must not keep the tail of the long bundle. + * The old in-place open without O_TRUNC kept that tail. + */ + + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%s/example.org%Z", dir); + (void) nxt_sprintf((u_char *) tmp_name, + (u_char *) tmp_name + sizeof(tmp_name), + "%s/.example.org.tmp%Z", dir); + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "-----BEGIN LONG BUNDLE-----") + != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the long bundle store failed"); + } + + if (nxt_main_file_store_test_store(task, dir, tmp_name, name, + "-----SHORT-----") != NXT_OK) + { + nxt_main_file_store_test_fail(thr, "the short bundle store failed"); + } + + if (nxt_main_file_store_test_holds(name, "-----SHORT-----") != NXT_OK) { + nxt_main_file_store_test_fail(thr, "\"%s\" kept a tail of the longer " + "bundle it replaced", name); + } + + if (stat(tmp_name, &st) == 0) { + nxt_main_file_store_test_fail(thr, "\"%s\" was left behind by a " + "bundle store", tmp_name); + } + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "main file store test passed"); diff --git a/test/test_tls.py b/test/test_tls.py index 6609dcce7..a8dd340cc 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -1,4 +1,6 @@ import io +import os +import shutil import ssl import subprocess import time @@ -6,6 +8,7 @@ import pytest +from conftest import unit_run, unit_stop from unit.applications.tls import ApplicationTLS from unit.option import option @@ -83,6 +86,12 @@ def generate_ca_conf(): Path(f'{option.temp_dir}/certindex.attr').touch() +def replace_cert(name='default'): + # Make a new key pair under the same name and store it over the old one. + client.certificate(name, False) + return client.certificate_load(name) + + def remove_tls(application='empty', port=8080): assert 'success' in client.conf( {"pass": f"applications/{application}"}, f'listeners/*:{port}' @@ -168,7 +177,6 @@ def test_tls_certificate_remove_nonexisting(): ), 'remove nonexistings certificate' -@pytest.mark.skip('not yet') def test_tls_certificate_update(): client.load('empty') @@ -178,12 +186,265 @@ def test_tls_certificate_update(): cert_old = ssl.get_server_certificate(('127.0.0.1', 8080)) - client.certificate() + # The listener names the bundle, so Unit applies the configuration + # again before it answers. + assert ( + replace_cert().get('success') == 'Certificate chain updated.' + ), 'replaced' assert cert_old != ssl.get_server_certificate( ('127.0.0.1', 8080) ), 'update certificate' + assert 'chain' in client.conf_get('/certificates/default'), 'listed' + + +def test_tls_certificate_update_unused(): + client.load('empty') + + client.certificate() + client.certificate('unused') + + add_tls() + + # No listener names the bundle, so there is no reconfiguration. + assert ( + replace_cert('unused').get('success') == 'Certificate chain uploaded.' + ), 'stored' + + +def test_tls_certificate_update_keepalive(): + client.load('empty') + + client.certificate() + + add_tls() + + (resp, sock) = client.get_ssl( + headers={'Host': 'localhost', 'Connection': 'keep-alive'}, + start=True, + read_timeout=1, + ) + + assert resp['status'] == 200, 'keepalive 1' + + cert_old = sock.getpeercert(True) + + assert 'success' in replace_cert(), 'replaced' + + # The accepted connection keeps its old TLS context. + (resp, sock) = client.get_ssl( + headers={'Host': 'localhost', 'Connection': 'close'}, + sock=sock, + start=True, + ) + + assert resp['status'] == 200, 'keepalive 2' + assert sock.getpeercert(True) == cert_old, 'old connection, old cert' + sock.close() + + (resp, sock) = client.get_ssl( + headers={'Host': 'localhost', 'Connection': 'close'}, start=True + ) + + assert resp['status'] == 200, 'new connection' + assert sock.getpeercert(True) != cert_old, 'new connection, new cert' + + +def test_tls_certificate_update_inflight(): + client.load('empty') + + client.certificate() + + add_tls() + + # Send half of the request before the replacement and the rest after. + sock = client.http( + b'GET / HTTP/1.1\r\nHost: localhost\r\n', + raw=True, + no_recv=True, + wrapper=client._default_context.wrap_socket, + ) + + cert_old = sock.getpeercert(True) + + assert 'success' in replace_cert(), 'replaced' + + resp = client.http(b'Connection: close\r\n\r\n', raw=True, sock=sock) + + assert resp['status'] == 200, 'in-flight request' + assert cert_old != ssl.get_server_certificate( + ('127.0.0.1', 8080) + ), 'new handshake, new cert' + + +def test_tls_certificate_update_mismatch(skip_alert): + skip_alert(r'certificate and private key do not match') + + client.load('empty') + + client.certificate() + + add_tls() + + cert_old = ssl.get_server_certificate(('127.0.0.1', 8080)) + + # Unit refuses a bundle with the wrong key. Nothing changes. + client.certificate('other', False) + + assert 'error' in client.certificate_load('default', 'other'), 'refused' + + assert cert_old == ssl.get_server_certificate( + ('127.0.0.1', 8080) + ), 'old certificate still served' + + assert 'chain' in client.conf_get('/certificates/default'), 'still listed' + + +def test_tls_certificate_update_store_fail(skip_alert): + skip_alert(r'unlink.*failed', r'failed to store certificate') + + client.load('empty') + + client.certificate() + + add_tls() + + cert_old = ssl.get_server_certificate(('127.0.0.1', 8080)) + info_old = client.conf_get('/certificates/default') + + # A directory with a file in it blocks the temporary file of main. + tmp = f'{option.temp_dir}/state/certs/.store.tmp' + os.makedirs(tmp) + Path(f'{tmp}/block').touch() + + assert ( + replace_cert()['error'] == 'Failed to store certificate.' + ), 'store failed' + + # The old metadata is back, and the old bundle stays in use. + assert client.conf_get('/certificates/default') == info_old, 'old info' + assert cert_old == ssl.get_server_certificate( + ('127.0.0.1', 8080) + ), 'old certificate still served' + + # A new name is removed from the metadata. + assert 'error' in replace_cert('new'), 'new store failed' + assert 'new' not in client.conf_get('/certificates'), 'no new info' + assert 'new' not in client.conf_get('/')['certificates'], 'not in root' + + shutil.rmtree(tmp) + + +def test_tls_certificate_update_long_name(): + client.certificate('default', False) + + with open(f'{option.temp_dir}/default.key', 'rb') as k, open( + f'{option.temp_dir}/default.crt', 'rb' + ) as c: + bundle = k.read() + c.read() + + # The longest file name on most file systems. The temporary file of + # main has a fixed name, so the store does not need a longer name. + name = 'a' * 255 + + for _ in range(2): + assert 'success' in client.conf( + bundle, f'/certificates/{name}' + ), 'stored' + + assert 'chain' in client.conf_get(f'/certificates/{name}'), 'listed' + + +def test_tls_certificate_update_sni(): + client.load('empty') + + client.certificate('default') + client.certificate('localhost') + + add_tls(cert=['default', 'localhost']) + + def peer_cert(host): + (resp, sock) = client.get_ssl( + headers={'Host': host, 'Connection': 'close'}, start=True + ) + + assert resp['status'] == 200, host + return sock.getpeercert(True) + + default_old = peer_cert('default') + localhost_old = peer_cert('localhost') + + assert default_old != localhost_old, 'sni selects the bundle' + + # Replace one bundle of the array. The other bundle does not change. + assert ( + replace_cert('localhost').get('success') + == 'Certificate chain updated.' + ), 'replaced' + + assert peer_cert('default') == default_old, 'other element untouched' + assert peer_cert('localhost') != localhost_old, 'element replaced' + + +def test_tls_certificate_update_restart(requires_restart): + client.certificate() + + # Use no application. unit_run() expects a restarted instance to have + # no applications. + assert 'success' in client.conf( + { + "listeners": { + "*:8080": {"pass": "routes", "tls": {"certificate": "default"}} + }, + "routes": [{"action": {"return": 200}}], + "applications": {}, + } + ) + + assert 'success' in replace_cert(), 'replaced' + + cert_new = ssl.get_server_certificate(('127.0.0.1', 8080)) + + temp_dir_old = option.temp_dir + statedir = f'{temp_dir_old}/state' + + # Main renamed the bundle into place. No temporary file is left. + assert [ + name for name in os.listdir(f'{statedir}/certs') if name[0] == '.' + ] == [], 'no temporary left' + + unit_stop() + + try: + # The fixture owns the new instance. The fixture stops it, checks + # its log, and removes its temp dir. This test removes the old one. + unit_run(state_dir=statedir) + + assert cert_new == ssl.get_server_certificate( + ('127.0.0.1', 8080) + ), 'replaced bundle survives a restart' + + assert 'chain' in client.conf_get('/certificates/default'), 'listed' + + finally: + unit_stop() + shutil.rmtree(temp_dir_old, ignore_errors=True) + + +def test_tls_certificate_too_large(): + client.load('empty') + + # The bundle is over the 1 MiB limit. The controller refuses it, not main. + resp = client.put( + **client._get_args('/certificates/big', b'-' * (1024 * 1024 + 1)) + ) + + assert resp['status'] == 413, resp['body'] + assert 'too large' in resp['body'], 'too large message' + + assert 'error' in client.conf_get('/certificates/big'), 'not stored' + def test_tls_certificate_key_incorrect(skip_alert): skip_alert(r'certificate and private key do not match') From 96f88f76c48c8aa67f4bb341dd0d9bd3070fbfd3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 16:57:38 +0200 Subject: [PATCH 288/391] build: keep the NXT_TESTS hooks out of unitd, libunit.a and modules A "hook" is a fault-injection branch or counter compiled in only for the C test suite, inside "#if (NXT_TESTS)". Today master has 22 such blocks in 10 core and libunit sources (src/nxt_application.c, src/nxt_main_process.c, src/nxt_mp.c, src/nxt_port.c, src/nxt_port_rpc.c, src/nxt_port_socket.c, src/nxt_random.c, src/nxt_router.c, src/nxt_socketpair.c, src/nxt_unit.c), 10 declaration blocks in 9 headers, and 2 blocks in src/test/nxt_unit_port_recv_test.c. They define 43 symbols: 28 in the core, 15 in libunit. "./configure --tests" made NXT_TESTS a global macro through auto/have. Everything built after it carried the hooks: unitd, libnxt.a, libunit.a and every language module. The deb and rpm packaging pass --tests in CONFIGURE_ARGS_COMMON (pkg/deb/Makefile:172, pkg/rpm/Makefile:145) and the Alpine aport passes it too, so the packages distros ship are the instrumented build. NXT_TESTS is now a per-object macro: * auto/sources: --tests no longer calls auto/have. The macro is absent from build/include/nxt_auto_config.h. Two archive names are added: NXT_LIB_TEST_STATIC=libnxt-tests.a and NXT_LIB_UNIT_TEST_STATIC=libunit-tests.a. * auto/make: with --tests, every source of NXT_LIB_SRCS and NXT_LIB_UNIT_SRCS is compiled a second time with -DNXT_TESTS=1 into build/src/.test.o and archived as libnxt-tests.a and libunit-tests.a. Every source under src/test/ is compiled with -DNXT_TESTS=1. Every program of the "tests" target links only the two -tests archives. unitd, libnxt.a, libunit.a and the modules are built from the plain objects and never see the macro. The plain build is not changed: "make" without "make tests" compiles nothing extra. The whole set is compiled twice, not a list of the sources that carry hooks. A list drifts: the first version of this change listed 8 sources, and master then added hooks to src/nxt_socketpair.c and src/nxt_unit.c. With the list, "make tests" on master fails to link (undefined reference to nxt_socketpair_test_send_fail, 4 sites in src/test/nxt_port_fail_test.c) and the libunit hooks ship. * .github/scripts/check-test-hooks.sh (new): derives the hook symbols from the build, as the symbols defined in .test.o and not in .o. It keeps no list of names or sources. It fails when NXT_TESTS is in nxt_auto_config.h or the derived set is empty; when a source outside src/test/ tests NXT_TESTS in a preprocessor directive but has no .test.o, or its two objects are the same after --strip-debug; when unitd, libnxt.a, libunit.a, build/src/nxt_unit.o, libnxt.so or a module defines a hook symbol; and when a test program that linked an object lacks that object's hooks (it took the plain copy). * .github/workflows/build-test.yml: the c-tests job builds unitd and libunit.a, then the tests, and runs the script before the tests. Cost, measured with gcc, -j8, "--openssl --tests --debug", this tree: "make" 4.45 s wall / 21.4 s CPU before and 5.58 s / 21.2 s after (wall is noise, the CPU time is equal); "make tests" 2.78 s / 9.8 s before and 9.80 s / 34.0 s after; 110 more objects; libnxt-tests.a is 8962360 B and libunit-tests.a 527588 B. Without --tests, "make" is unchanged: 4.14 s, no .test.o. Verified: * ./configure --openssl --tests --debug; make; make tests: ./build/tests, ./build/unit_close_test and ./build/unit_port_recv_test exit 0. A second "make" and "make tests" rebuild nothing. * The gate passes. unitd, libnxt.a, libunit.a, build/src/nxt_unit.o and build/lib/unit/modules/php.unit.so define 0 of the 43 hook symbols, in the --tests build and in a plain "./configure --openssl" build. build/tests defines all 28 core hooks; unit_port_recv_test defines all 15 libunit hooks, and its object references 13 of them, so the "#if (NXT_TESTS)" case in that file is compiled in. * "make tests" without --tests still refuses. * pytest test/test_app_lifecycle.py -k libunit: 5 passed, 40 deselected. build/unit_app_test now links libunit-tests.a, which is what the global macro gave it before, so the libunit runtime is unchanged. * Negative controls, each makes the gate exit 1: a hook added to src/nxt_main.c (compiled once); the auto/have line restored (macro in nxt_auto_config.h, empty hook set); build/unit_app_test relinked against plain libunit.a (15 hook symbols missing); nxt_mp.test.o copied over nxt_mp.o (identical objects). Packaging needs no change. deb, rpm and the Alpine aport keep --tests; their unitd, libunit.a and modules are now free of hooks. deb and rpm do not run the C suite (DOTESTS=0, dotests 0); the aport runs ./build/tests in check() as before. One side effect is gone: "make tests" no longer builds the plain libunit.a on the way, because unit_app_test depends on libunit-tests.a now. deb and rpm run "make build/lib/libunit.a" explicitly, and libunit-install and the go module depend on libunit.a, so each still gets it. Not done here: nxt_mp_test_sizes() is not inside "#if (NXT_TESTS)", so it is not a hook by this definition and stays exported in every build. A hook written as inline code in a header can reach a source that is compiled once (src/nxt_main.c, module sources); today every header block holds declarations only, and the gate does not cover that case. Runtime failpoints and release-build probes are separate work. The deb and rpm packages pass --tests, so every released package from 0.2 to 1.36.1 was built this way (checked per tag): all of them carried nxt_random_test(), and 1.36.0 and 1.36.1 the fault-injection hooks in nxt_mp.c, nxt_port_rpc.c and nxt_port_socket.c. CHANGES records this as a change to the shipped artifacts. Co-Authored-By: Claude Fable 5.1 --- .github/scripts/check-test-hooks.sh | 271 ++++++++++++++++++++++++++++ .github/workflows/build-test.yml | 17 ++ CHANGES | 10 + auto/make | 115 ++++++++++-- auto/sources | 16 +- docs/changes.xml | 13 ++ 6 files changed, 418 insertions(+), 24 deletions(-) create mode 100755 .github/scripts/check-test-hooks.sh diff --git a/.github/scripts/check-test-hooks.sh b/.github/scripts/check-test-hooks.sh new file mode 100755 index 000000000..e5a56f6e2 --- /dev/null +++ b/.github/scripts/check-test-hooks.sh @@ -0,0 +1,271 @@ +#!/bin/sh +# +# Check that the NXT_TESTS hooks reach the test programs and nothing else. +# +# A "hook" is a fault-injection branch or counter compiled into the core or +# into libunit only for the C test suite, inside "#if (NXT_TESTS)". With +# "./configure --tests", "make tests" compiles every core and libunit source +# twice: build/src/X.o without the macro and build/src/X.test.o with it +# (auto/make). unitd, libnxt.a, libunit.a and the language modules are built +# from the plain objects; the test programs link libnxt-tests.a and +# libunit-tests.a, built from the instrumented ones. +# +# The script derives everything from the build. It keeps no list of hook +# names or hooked sources, so a new hook needs no change here. +# +# hook symbols: the symbols defined in X.test.o and not in X.o. Names that +# contain a "." are compiler-made clones (foo.isra.0, foo.cold) +# and are skipped; a hook is a C identifier. +# members: the global symbols of X.test.o. A program that defines one +# of them linked object X, from one of the two archives. +# +# Checks, in order: +# +# 0. NXT_TESTS is not in build/include/nxt_auto_config.h, and at least one +# hook symbol exists. If the macro leaked into the plain objects, X.o and +# X.test.o are the same and the hook set is empty; that is a failure, not +# a pass. +# 1. Every src/**/*.c outside src/test/ that mentions NXT_TESTS has a +# build/src/X.test.o, and that object differs from X.o once debug info +# is removed. A hook in a source that is compiled once (for example +# src/nxt_main.c or a module source) would ship; a plain object that +# equals its instrumented twin was compiled with the macro. +# 2. No shipped artifact defines a hook symbol: build/sbin/unitd, +# build/lib/libnxt.a, build/lib/libunit.a, build/src/nxt_unit.o (the +# object the language modules link), and, when present, build/lib/libnxt.so +# and every build/*.unit.so. +# 3. Every test program that linked object X defines every hook of X. This +# proves it took X from the instrumented archive, not the plain one. +# +# Usage: .github/scripts/check-test-hooks.sh [build-dir] +# +# Run it after "./configure --tests", "make" and "make tests". It needs +# nm(1), objcopy(1) and cmp(1). + +set -eu + +# sort(1) and comm(1) must agree on the collation order +LC_ALL=C +export LC_ALL + +build=${1:-build} + +tmp=$(mktemp -d) +finished= + +# With "set -e" a failing nm, comm or awk would end the script without a +# word; say so, and only then is a silent exit impossible. +finish() +{ + status=$? + rm -rf "$tmp" + + if [ -z "$finished" ] && [ $status -ne 0 ]; then + echo "::error::check-test-hooks.sh aborted with status $status" >&2 + fi +} + +trap finish EXIT + +fail=0 + +error() +{ + echo "::error::$*" >&2 + fail=1 +} + +# defined symbol names of one file (an object, an archive or a program) +names() +{ + nm --defined-only -p "$1" 2>/dev/null | awk 'NF >= 3 { print $NF }' \ + | sort -u +} + +# defined global symbol names of one object +global_names() +{ + nm --defined-only -p "$1" 2>/dev/null \ + | awk 'NF >= 3 && $(NF - 1) ~ /^[A-Z]$/ { print $NF }' | sort -u +} + + +# 0. The macro is per object, and the derivation works. + +if grep -q 'NXT_TESTS' "$build/include/nxt_auto_config.h"; then + error "NXT_TESTS is defined in $build/include/nxt_auto_config.h;" \ + "it must stay a per-object macro (see auto/make)" +fi + +test_objs=$(find "$build/src" -name '*.test.o' | sort) + +if [ -z "$test_objs" ]; then + error "no *.test.o under $build/src; run 'make tests' first" + finished=1 + exit 1 +fi + +: > "$tmp/hooks" + +for t in $test_objs; do + p=${t%.test.o}.o + + if [ ! -f "$p" ]; then + error "$t has no plain object $p" + continue + fi + + names "$p" > "$tmp/plain" + names "$t" > "$tmp/instr" + + # symbols only the instrumented object defines + comm -13 "$tmp/plain" "$tmp/instr" | grep -v '\.' > "$tmp/hooks.$$" || true + + if [ -s "$tmp/hooks.$$" ]; then + rel=${t#"$build"/} + rel=${rel%.test.o} + awk -v o="$rel" '{ print o, $0 }' "$tmp/hooks.$$" >> "$tmp/hooks" + fi +done + +nhooks=$(wc -l < "$tmp/hooks") + +if [ "$nhooks" -eq 0 ]; then + error "no symbol differs between the plain and the instrumented objects;" \ + "NXT_TESTS leaked into the plain build or is missing from the" \ + "instrumented one" + finished=1 + exit 1 +fi + +echo "hook symbols: $nhooks" +sed 's/^/ /' "$tmp/hooks" + + +# 1. Every hooked source is compiled twice, and the two objects differ. +# +# The symbol diff above cannot see a hook that adds no symbol, and it goes +# empty for an object that was compiled with the macro both times. So for +# every source that tests NXT_TESTS in a preprocessor directive the plain and +# the instrumented object, debug info removed, must not be the same file. A +# mention in a comment is not a hook and is not counted. + +for src in $(find src -name '*.c' -not -path 'src/test/*' | sort); do + grep -Eq '^[[:space:]]*#[[:space:]]*(if|ifdef|ifndef|elif)\b.*NXT_TESTS' \ + "$src" || continue + + p=$build/${src%.c}.o + t=$build/${src%.c}.test.o + + if [ ! -f "$t" ]; then + error "$src tests NXT_TESTS but is compiled once, without the" \ + "macro; its hooks would ship or never link. Hooks belong in" \ + "a source of NXT_LIB_SRCS or NXT_LIB_UNIT_SRCS (auto/sources)" + continue + fi + + objcopy --strip-debug "$p" "$tmp/plain.o" + objcopy --strip-debug "$t" "$tmp/instr.o" + + if cmp -s "$tmp/plain.o" "$tmp/instr.o"; then + error "$src tests NXT_TESTS but $p and $t are the same object;" \ + "the plain object was compiled with the macro" + fi +done + + +# 2. Shipped artifacts carry no hook symbol. +# +# A static function that the plain compile inlines and the instrumented one +# does not is in the hook set too. Another plain object may define a static +# of the same name, and that name is then legitimately in unitd. Such names +# cannot be checked by name; they are reported and left out. This hides no +# leak: a plain object compiled with the macro is caught by check 0, because +# auto/make has one rule template for all plain objects. + +awk '{ print $2 }' "$tmp/hooks" | sort -u > "$tmp/hooknames" + +for t in $test_objs; do + names "${t%.test.o}.o" +done | sort -u > "$tmp/plainnames" + +ambiguous=$(comm -12 "$tmp/hooknames" "$tmp/plainnames") + +if [ -n "$ambiguous" ]; then + echo "not checked by name, defined by another plain object:" \ + "$(echo $ambiguous)" + comm -23 "$tmp/hooknames" "$tmp/plainnames" > "$tmp/hooknames.$$" + mv "$tmp/hooknames.$$" "$tmp/hooknames" +fi + +shipped="$build/sbin/unitd $build/lib/libnxt.a $build/lib/libunit.a \ + $build/src/nxt_unit.o" + +if [ -f "$build/lib/libnxt.so" ]; then + shipped="$shipped $build/lib/libnxt.so" +fi + +shipped="$shipped $(find "$build" -name '*.unit.so' | sort)" + +for f in $shipped; do + if [ ! -f "$f" ]; then + error "$f is missing; run 'make', 'make $build/lib/libunit.a' and" \ + "'make tests' first" + continue + fi + + names "$f" > "$tmp/defined" + + leaked=$(comm -12 "$tmp/hooknames" "$tmp/defined") + + if [ -n "$leaked" ]; then + error "$f defines NXT_TESTS hook symbols: $(echo $leaked)" + fi +done + + +# 3. The test programs took every linked object from the instrumented archive. + +programs="$build/tests $build/ncq_test $build/vbcq_test $build/unit_app_test \ + $build/unit_close_test $build/unit_port_recv_test \ + $build/unit_websocket_chat $build/unit_websocket_echo" + +for prog in $programs; do + if [ ! -f "$prog" ]; then + error "$prog is missing; run 'make tests' first" + continue + fi + + names "$prog" > "$tmp/defined" + + for t in $test_objs; do + rel=${t#"$build"/} + rel=${rel%.test.o} + + awk -v o="$rel" '$1 == o { print $2 }' "$tmp/hooks" | sort -u \ + > "$tmp/want" + + [ -s "$tmp/want" ] || continue + + global_names "$t" > "$tmp/members" + + # not linked at all: nothing to prove + [ -n "$(comm -12 "$tmp/members" "$tmp/defined")" ] || continue + + missing=$(comm -23 "$tmp/want" "$tmp/defined") + + if [ -n "$missing" ]; then + error "$prog linked $rel from the plain archive:" \ + "missing hook symbols $(echo $missing)" + fi + done +done + + +finished=1 + +if [ $fail -ne 0 ]; then + exit 1 +fi + +echo "ok: hooks are in the test programs only" diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6c3c4c4de..6e97e446e 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -11,6 +11,7 @@ on: - 'pkg/contrib/**' - 'pkg/eol.json' - '.github/workflows/build-test.yml' + - '.github/scripts/check-test-hooks.sh' push: branches: master paths: @@ -22,6 +23,7 @@ on: - 'pkg/contrib/**' - 'pkg/eol.json' - '.github/workflows/build-test.yml' + - '.github/scripts/check-test-hooks.sh' env: # OpenSSL patch version built from source (see "Build OpenSSL" step). Only the @@ -857,9 +859,24 @@ jobs: - name: Configure unit (--tests) run: ./configure --openssl --tests + - name: Build unitd and libunit + run: | + make -j4 + make -j4 build/lib/libunit.a + - name: Build C tests run: make -j4 tests + # The hooks (fault-injection branches and counters inside + # "#if (NXT_TESTS)") must reach the test programs and nothing else. + # The script derives the hook symbols from the build itself, so a new + # hook needs no change here. It fails when a hook symbol is in unitd, + # libnxt.a, libunit.a or nxt_unit.o, when a test program linked a plain + # object, or when a source outside the twice-compiled set mentions + # NXT_TESTS. + - name: Check the test hooks are in the test programs only + run: .github/scripts/check-test-hooks.sh + - name: Run C tests run: sudo ./build/tests diff --git a/CHANGES b/CHANGES index 7be950b51..2858a7ff8 100644 --- a/CHANGES +++ b/CHANGES @@ -481,6 +481,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 applied to the next message. + *) Change: unitd, libunit.a and the language modules no longer contain + the test hooks compiled under "#if (NXT_TESTS)". "./configure + --tests" used to define the macro for the whole build, and the deb + and rpm packages pass "--tests", so every released package from 0.2 + to 1.36.1 carried them: the nxt_random_test() entry point in all of + them, and in 1.36.0 and 1.36.1 the fault-injection branches and + counters in nxt_mp.c, nxt_port_rpc.c and nxt_port_socket.c, two of + which were exported symbols. The hooks now go only into the test + programs. A build without "--tests" is unchanged. + Changes with FreeUnit 1.36.1 28 Aug 2026 *) Feature: configuration validation errors now say where they are. A diff --git a/auto/make b/auto/make index 509eaaa66..1cc26788f 100644 --- a/auto/make +++ b/auto/make @@ -185,11 +185,21 @@ do nxt_dep=${nxt_src%.c}.dep nxt_dep_flags=`nxt_gen_dep_flags` nxt_dep_post=`nxt_gen_dep_post` + + # Only the test sources see NXT_TESTS here. The core and libunit + # objects built by this loop are the ones unitd, libnxt.a, libunit.a + # and the language modules link, so they never get the macro. The + # instrumented copies the test programs link are built further down. + case "$nxt_src" in + src/test/*) nxt_test_cflags="-DNXT_TESTS=1" ;; + *) nxt_test_cflags= ;; + esac + cat << END >> $NXT_MAKEFILE $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H \$(PP_CC) \$@ - \$(v)\$(CC) -c \$(CFLAGS) \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ + \$(v)\$(CC) -c \$(CFLAGS) $nxt_test_cflags \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ -o $NXT_BUILD_DIR/$nxt_obj \\ $nxt_dep_flags \\ $nxt_src @@ -210,7 +220,7 @@ cat << END >> $NXT_MAKEFILE $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H \$(PP_CC) \$@ - \$(v)\$(CC) -c \$(CFLAGS) -DNXT_NCQ_TEST=1 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ + \$(v)\$(CC) -c \$(CFLAGS) -DNXT_TESTS=1 -DNXT_NCQ_TEST=1 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ -o $NXT_BUILD_DIR/$nxt_obj \\ $nxt_dep_flags \\ $nxt_src @@ -229,7 +239,7 @@ cat << END >> $NXT_MAKEFILE $NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H \$(PP_CC) \$@ - \$(v)\$(CC) -c \$(CFLAGS) -DNXT_NCQ_TEST=0 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ + \$(v)\$(CC) -c \$(CFLAGS) -DNXT_TESTS=1 -DNXT_NCQ_TEST=0 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ -o $NXT_BUILD_DIR/$nxt_obj \\ $nxt_dep_flags \\ $nxt_src @@ -244,6 +254,73 @@ $echo >> $NXT_MAKEFILE if [ $NXT_TESTS = YES ]; then + # Instrumented core and libunit. + # + # Every source of libnxt.a and libunit.a is compiled a second time with + # -DNXT_TESTS=1 into .test.o and archived as libnxt-tests.a and + # libunit-tests.a. The test programs link only these archives; the plain + # archives above are what unitd and the language modules link. Compiling + # the whole set, not a list of the sources that carry hooks, means a hook + # added to any core or libunit source is covered without a build change. + + $echo "NXT_LIB_TEST_OBJS = \\" >> $NXT_MAKEFILE + + for nxt_src in $NXT_LIB_SRCS + do + nxt_obj=${nxt_src%.c}.test.o + $echo " $NXT_BUILD_DIR/$nxt_obj \\" >> $NXT_MAKEFILE + done + + $echo >> $NXT_MAKEFILE + + $echo "NXT_LIB_UNIT_TEST_OBJS = \\" >> $NXT_MAKEFILE + $echo " $NXT_BUILD_DIR/src/nxt_lvlhsh.test.o \\" >> $NXT_MAKEFILE + $echo " $NXT_BUILD_DIR/src/nxt_murmur_hash.test.o \\" >> $NXT_MAKEFILE + $echo " $NXT_BUILD_DIR/src/nxt_socket_msg.test.o \\" >> $NXT_MAKEFILE + $echo " $NXT_BUILD_DIR/src/nxt_websocket.test.o \\" >> $NXT_MAKEFILE + + for nxt_src in $NXT_LIB_UNIT_SRCS + do + nxt_obj=${nxt_src%.c}.test.o + $echo " $NXT_BUILD_DIR/$nxt_obj \\" >> $NXT_MAKEFILE + done + + $echo >> $NXT_MAKEFILE + + for nxt_src in $NXT_LIB_SRCS $NXT_LIB_UNIT_SRCS + do + nxt_obj=${nxt_src%.c}.test.o + nxt_dep=${nxt_src%.c}.test.dep + nxt_dep_flags=`nxt_gen_dep_flags` + nxt_dep_post=`nxt_gen_dep_post` + cat << END >> $NXT_MAKEFILE + +$NXT_BUILD_DIR/$nxt_obj: $nxt_src $NXT_VERSION_H + \$(PP_CC) \$@ + \$(v)\$(CC) -c \$(CFLAGS) -DNXT_TESTS=1 \$(NXT_LIB_INCS) $NXT_LIB_AUX_CFLAGS \\ + -o $NXT_BUILD_DIR/$nxt_obj \\ + $nxt_dep_flags \\ + $nxt_src + $nxt_dep_post + +-include $NXT_BUILD_DIR/$nxt_dep + +END + + done + + cat << END >> $NXT_MAKEFILE + +$NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC: \$(NXT_LIB_TEST_OBJS) + \$(PP_AR) \$@ + \$(v)$NXT_STATIC_LINK \$@ \$(NXT_LIB_TEST_OBJS) + +$NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC: \$(NXT_LIB_UNIT_TEST_OBJS) + \$(PP_AR) \$@ + \$(v)$NXT_STATIC_LINK \$@ \$(NXT_LIB_UNIT_TEST_OBJS) + +END + # Test object files list. $echo "NXT_TEST_OBJS = \\" >> $NXT_MAKEFILE @@ -268,73 +345,73 @@ tests: $NXT_BUILD_DIR/tests \\ $NXT_BUILD_DIR/unit_port_recv_test $NXT_BUILD_DIR/tests: \$(NXT_TEST_OBJS) \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \\ \$(NXT_OTEL_LIB_STATIC) \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/tests \\ \$(CFLAGS) \$(NXT_TEST_OBJS) \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \$(NXT_OTEL_LIB_STATIC) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/ncq_test: $NXT_BUILD_DIR/src/test/nxt_ncq_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \\ \$(NXT_OTEL_LIB_STATIC) \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/ncq_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_ncq_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \$(NXT_OTEL_LIB_STATIC) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/vbcq_test: $NXT_BUILD_DIR/src/test/nxt_vbcq_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \\ \$(NXT_OTEL_LIB_STATIC) \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/vbcq_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_vbcq_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(NXT_OTEL_LIB_STATIC) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \$(NXT_OTEL_LIB_STATIC) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_app_test: $NXT_BUILD_DIR/src/test/nxt_unit_app_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_app_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_app_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_close_test: $NXT_BUILD_DIR/src/test/nxt_unit_close_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_close_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_close_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_port_recv_test: \\ $NXT_BUILD_DIR/src/test/nxt_unit_port_recv_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_port_recv_test \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_port_recv_test.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_websocket_chat: \\ $NXT_BUILD_DIR/src/test/nxt_unit_websocket_chat.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_websocket_chat \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_websocket_chat.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS $NXT_BUILD_DIR/unit_websocket_echo: \\ $NXT_BUILD_DIR/src/test/nxt_unit_websocket_echo.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \$(PP_LD) \$@ \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_websocket_echo \\ \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_websocket_echo.o \\ - $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS END diff --git a/auto/sources b/auto/sources index b0fd84fbc..f1ea56685 100644 --- a/auto/sources +++ b/auto/sources @@ -109,6 +109,17 @@ NXT_LIB_SRCS=" \ NXT_LIB_UNIT_SRCS="src/nxt_unit.c" +# The C test suite needs the NXT_TESTS hooks: fault-injection branches and +# counters compiled into the core and into libunit. NXT_TESTS is never a +# global macro. "make tests" compiles every core and libunit source a second +# time with -DNXT_TESTS=1 into these two archives, and only the test programs +# link them. unitd, libnxt.a, libunit.a and the language modules are built +# from the plain objects and never see the macro. +# +NXT_LIB_TEST_STATIC="libnxt-tests.a" +NXT_LIB_UNIT_TEST_STATIC="libunit-tests.a" + + NXT_LIB_TLS_DEPS="src/nxt_tls.h" NXT_LIB_TLS_SRCS="src/nxt_cert.c" NXT_LIB_OPENSSL_SRCS="src/nxt_openssl.c" @@ -329,11 +340,6 @@ if [ "$NXT_TEST_BUILD" = "YES" ]; then fi -if [ $NXT_TESTS = YES ]; then - nxt_have=NXT_TESTS . auto/have -fi - - NXT_FUZZ_SRCS=" \ fuzzing/nxt_basic_fuzz.c \ fuzzing/nxt_http_controller_fuzz.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 9ea5d87d6..cf068ab7e 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -633,6 +633,19 @@ limit applied to the next message. + + +unitd, libunit.a and the language modules no longer contain the test hooks +compiled under "#if (NXT_TESTS)". "./configure --tests" used to define the +macro for the whole build, and the deb and rpm packages pass "--tests", so +every released package from 0.2 to 1.36.1 carried them: the nxt_random_test() +entry point in all of them, and in 1.36.0 and 1.36.1 the fault-injection +branches and counters in nxt_mp.c, nxt_port_rpc.c and nxt_port_socket.c, two +of which were exported symbols. The hooks now go only into the test programs. +A build without "--tests" is unchanged. + + + From 4468477a7dc0a6110060b36393856747c43f41ba Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 16:43:05 +0200 Subject: [PATCH 289/391] changelog: a request length that does not fit the app protocol is refused The entry for https://github.com/freeunitorg/freeunit/pull/494 (593f9f5c): a method over 255 bytes now answers 501, and a header name that does not fit in one byte with the "HTTP_" prefix answers 431. Before, the length wrapped and the application read a different string from the one copied. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Gemini 3.8 Flash --- CHANGES | 9 +++++++++ docs/changes.xml | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGES b/CHANGES index 2858a7ff8..960e14ef4 100644 --- a/CHANGES +++ b/CHANGES @@ -21,6 +21,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 process was then linked into its parent's list twice, which corrupted the list in main. Such a message is now refused. + *) Bugfix: a request could reach the application with a truncated method + or header name length. The application protocol stores method and + field name lengths in a single byte. When a method exceeded 255 + bytes, the router wrapped the length. For PHP, Perl, and Ruby, header + names carry the "HTTP_" prefix, causing names of 251 to 255 bytes to + wrap. The router now refuses a method longer than 255 bytes with 501, + and an overflowing header name with 431. Both are logged at "info". + Other languages continue to accept header names up to 255 bytes. + *) Bugfix: an upstream 1xx interim response ("100 Continue", "103 Early Hints") was relayed to the client as the response, with the final response relayed as its body. The proxy now drops interim responses diff --git a/docs/changes.xml b/docs/changes.xml index cf068ab7e..e4aa2d400 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -40,6 +40,18 @@ list twice, which corrupted the list in main. Such a message is now refused. + + +a request could reach the application with a truncated method or header name +length. The application protocol stores method and field name lengths in a +single byte. When a method exceeded 255 bytes, the router wrapped the length. +For PHP, Perl, and Ruby, header names carry the "HTTP_" prefix, causing names +of 251 to 255 bytes to wrap. The router now refuses a method longer than +255 bytes with 501, and an overflowing header name with 431. Both are logged +at "info". Other languages continue to accept header names up to 255 bytes. + + + an upstream 1xx interim response ("100 Continue", "103 Early Hints") was From 1e73d92d04c63441a240e46551228dc7fad77270 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 21:31:50 +0000 Subject: [PATCH 290/391] build: add --hardening and the checked/span helpers Add a --hardening=[off|default|strict] configure option. The default is "off". A build that does not pass the option does not change. "default" adds the stack protector, _FORTIFY_SOURCE=3, stack-clash protection and control-flow protection. At link time it adds RELRO, BIND_NOW and noexecstack. "strict" also adds -Wformat=2, -Wvla, -Wimplicit-fallthrough and -ftrivial-auto-var-init=pattern. Each flag is probed first. An older compiler only loses that flag. The probes use the flags in the same order as the build. CFLAGS from the environment comes after the built-in -O. Thus CFLAGS=-O0 does not pass the FORTIFY probe with fortify silently off. The FORTIFY probe also checks __OPTIMIZE__ in its source, because modern glibc does not warn at -O0 any more. The probe includes . Thus the glibc warning "level 3 treated as 2" fails the probe, and not the real sources under -Werror. When level 3 is refused, the probe falls back to _FORTIFY_SOURCE=2. -fcf-protection is probed on every platform, not only on x86_64. Thus 32-bit x86 also gets control-flow protection. Each compiler rejects the flag for a target that it does not apply to. Thus the probe selects the correct flag on its own. GNU ld only warns about a -z keyword that it does not know, and it still links. Thus a probe for a -z flag was always "found". Now the script probes -Wl,--fatal-warnings first. When the linker accepts it, the linker probes use it. Then a flag that only gives a warning is not added. macOS ld does not accept the flag. There the probes run as before. -Wimplicit-fallthrough under Clang does not accept a "Fall through." comment. Add nxt_fallthrough. It is an __attribute__((fallthrough)) when the compiler has it (probed in auto/clang). Use it at every switch fall-through in place of the comment. Add two small headers for the later hardening changes: - nxt_checked.h: nxt_size_add() and nxt_size_mul(). They do size_t arithmetic and report an overflow instead of a wrap; - nxt_span.h: nxt_span_t, a [pos, end) view of untrusted bytes. nxt_span_take() and nxt_span_copy() refuse a record that does not fit in the bytes that are left. src/test/nxt_checked_test.c covers both headers. It is the first entry of a table of tests in nxt_tests.c. The later changes add to this table. nxt_test_in_child() and NXT_TEST_CHECK() are the helpers that those tests share. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 5 ++ auto/cc/hardening | 158 ++++++++++++++++++++++++++++++++++++ auto/clang | 19 +++++ auto/help | 10 +++ auto/options | 4 + auto/sources | 3 + auto/summary | 1 + configure | 1 + docs/changes.xml | 9 ++ src/nxt_checked.h | 31 +++++++ src/nxt_clang.h | 13 +++ src/nxt_h1proto.c | 2 +- src/nxt_http_parse.c | 12 +-- src/nxt_http_static.c | 2 +- src/nxt_murmur_hash.c | 4 +- src/nxt_router.c | 2 +- src/nxt_span.h | 93 +++++++++++++++++++++ src/nxt_string.c | 4 +- src/nxt_timer.c | 2 +- src/test/nxt_checked_test.c | 57 +++++++++++++ src/test/nxt_lvlhsh_test.c | 2 +- src/test/nxt_tests.c | 45 ++++++++++ src/test/nxt_tests.h | 12 +++ 23 files changed, 476 insertions(+), 15 deletions(-) create mode 100644 auto/cc/hardening create mode 100644 src/nxt_checked.h create mode 100644 src/nxt_span.h create mode 100644 src/test/nxt_checked_test.c diff --git a/CHANGES b/CHANGES index 960e14ef4..205224c5d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Feature: the --hardening=[off|default|strict] configure option adds + compiler and linker hardening flags. Each flag is probed, and a + compiler that does not support a flag only loses that flag. The + default is "off", so a build without the option does not change. + *) Feature: "PUT /certificates/" replaces an existing certificate bundle in place. Main writes the bundle to a temporary file and renames it over the old one, so a crash never leaves a partial diff --git a/auto/cc/hardening b/auto/cc/hardening new file mode 100644 index 000000000..b83c9bd36 --- /dev/null +++ b/auto/cc/hardening @@ -0,0 +1,158 @@ + +# Copyright (C) FreeUnit Community + + +# --hardening=[off|default|strict] (auto/options). The default is "off". +# With "off", NXT_CFLAGS and NXT_LD_OPT stay as auto/cc/test left them. +# A build that does not pass the option does not change. +# +# Each flag below is probed before it is added. The probe compiles a small +# program with the flag under -Werror. For a linker flag, the probe also +# links the program. When an older compiler, a different target or a libc +# does not support a flag, the build does not fail. It only does not get +# that one flag. This follows the "try it" approach of auto/feature. The +# value and run reporting of auto/feature is not used, because these flags +# do not need it. + +case "$NXT_HARDENING" in + off | default | strict) + ;; + + *) + $echo + $echo $0: error: invalid value \"$NXT_HARDENING\" for --hardening. + $echo Valid values are: off, default, strict. + $echo + exit 1 + ;; +esac + + +if [ $NXT_HARDENING != off ]; then + +# The flags are in the order that auto/make uses. The flag under test goes +# into NXT_CFLAGS. CFLAGS from the environment comes last. Thus a -O0 in +# CFLAGS wins over the built-in -O in the probe, as it does in the build. +# The probe includes a libc header. With it, the #warning of glibc about +# a _FORTIFY_SOURCE level that it treats as a lower one fails the probe, +# and not the real sources. $2 is extra source for the probe. +nxt_hardening_try_cflag() +{ + nxt_flag=$1 + + $echo -n "checking for $nxt_flag ..." + + cat << END > $NXT_AUTOTEST.c +#include +${2-} +int main(void) { return 0; } +END + + if /bin/sh -c "($CC $NXT_CFLAGS $nxt_flag $NXT_CC_OPT $CFLAGS -Werror \ + -o $NXT_AUTOTEST $NXT_AUTOTEST.c)" \ + >> $NXT_AUTOCONF_ERR 2>&1 + then + $echo " found" + NXT_CFLAGS="$NXT_CFLAGS $nxt_flag" + nxt_ret=0 + else + $echo " not found" + nxt_ret=1 + fi + + rm -rf $NXT_AUTOTEST $NXT_AUTOTEST.c + + return $nxt_ret +} + + +# Links a small program with the given linker flags. The exit status +# tells if the link succeeded. This function adds nothing to NXT_LD_OPT. +nxt_hardening_link() +{ + cat << END > $NXT_AUTOTEST.c +int main(void) { return 0; } +END + + /bin/sh -c "($CC $CFLAGS $NXT_CFLAGS $NXT_CC_OPT \ + -o $NXT_AUTOTEST $NXT_AUTOTEST.c $NXT_LD_OPT $*)" \ + >> $NXT_AUTOCONF_ERR 2>&1 + nxt_ret=$? + + rm -rf $NXT_AUTOTEST $NXT_AUTOTEST.c + + return $nxt_ret +} + + +# GNU ld only warns about a -z keyword that it does not know, and it links +# anyway. Thus a plain probe would "find" every -z flag. When the linker +# accepts -Wl,--fatal-warnings, the probes below run with it. Then a +# linker warning fails the probe. macOS ld does not accept the flag. +# There the probes run without it. +nxt_hardening_ld_fatal= + +nxt_hardening_try_ldflag() +{ + nxt_flag=$1 + + $echo -n "checking for $nxt_flag ..." + + if nxt_hardening_link $nxt_hardening_ld_fatal $nxt_flag; then + $echo " found" + NXT_LD_OPT="$NXT_LD_OPT $nxt_flag" + else + $echo " not found" + fi +} + + +# Do not stop on an error exit status while the probes run, the same as in +# auto/cc/test. A probe that fails to compile is expected and handled. It +# must not abort the whole configure run. The "set -e" after the last +# probe restores the default. +set +e + +nxt_hardening_try_cflag -fstack-protector-strong +nxt_hardening_try_cflag -fstack-clash-protection + +# Each flag applies to one target only: -fcf-protection to x86, and +# -mbranch-protection=standard to arm64. The compiler rejects the flag on +# the other target. Thus the probe alone selects the correct one. +nxt_hardening_try_cflag -fcf-protection +nxt_hardening_try_cflag -mbranch-protection=standard + +# _FORTIFY_SOURCE does nothing without optimization, and not every libc +# warns about that. Thus the probe checks the optimization level itself. +# When the libc treats a level as a lower one, it gives a warning. Then +# the probe tries the lower level. +nxt_fortify_check='#ifndef __OPTIMIZE__ +#error _FORTIFY_SOURCE needs an optimizing build +#endif' + +nxt_hardening_try_cflag -D_FORTIFY_SOURCE=3 "$nxt_fortify_check" \ + || nxt_hardening_try_cflag -D_FORTIFY_SOURCE=2 "$nxt_fortify_check" + +$echo -n "checking for -Wl,--fatal-warnings ..." + +if nxt_hardening_link -Wl,--fatal-warnings; then + $echo " found" + nxt_hardening_ld_fatal=-Wl,--fatal-warnings +else + $echo " not found" +fi + +nxt_hardening_try_ldflag -Wl,-z,relro +nxt_hardening_try_ldflag -Wl,-z,now +nxt_hardening_try_ldflag -Wl,-z,noexecstack + +if [ $NXT_HARDENING = strict ]; then + nxt_hardening_try_cflag -Wformat=2 + nxt_hardening_try_cflag -Wvla + nxt_hardening_try_cflag -Wimplicit-fallthrough + nxt_hardening_try_cflag -ftrivial-auto-var-init=pattern +fi + +set -e + +fi diff --git a/auto/clang b/auto/clang index fea8602ad..87ab8fafc 100644 --- a/auto/clang +++ b/auto/clang @@ -196,6 +196,25 @@ nxt_feature_test="static void f(void) __attribute__ ((__unused__)); . auto/feature +nxt_feature="GCC __attribute__ fallthrough" +nxt_feature_name=NXT_HAVE_GCC_ATTRIBUTE_FALLTHROUGH +nxt_feature_run= +nxt_feature_incs= +nxt_feature_libs= +nxt_feature_test="int main(int argc, char **argv) { + switch (argc) { + case 1: + argc++; + __attribute__ ((__fallthrough__)); + default: + break; + } + + return !argc; + }" +. auto/feature + + nxt_feature="GCC __attribute__ nonstring" nxt_feature_name=NXT_HAVE_GCC_ATTRIBUTE_NONSTRING nxt_feature_run= diff --git a/auto/help b/auto/help index 4e10d2844..c851e75b1 100644 --- a/auto/help +++ b/auto/help @@ -61,6 +61,16 @@ cat << END --debug enable debug logging + --hardening=MODE add compiler and linker hardening flags, probed for + compiler support (default: off, unchanged build): + off - no extra flags (default) + default - stack protector, _FORTIFY_SOURCE=3 (or 2), + stack-clash and control-flow protection, + RELRO/BIND_NOW/noexecstack at link time + strict - default, plus -Wformat=2, -Wvla, + -Wimplicit-fallthrough and + -ftrivial-auto-var-init=pattern + --fuzz=ENGINE enable fuzz testing diff --git a/auto/options b/auto/options index bb7beda17..78f8f0d5b 100644 --- a/auto/options +++ b/auto/options @@ -11,6 +11,8 @@ NXT_LD_OPT= NXT_DEBUG=NO +NXT_HARDENING=off + NXT_INET6=YES NXT_UNIX_DOMAIN=YES @@ -102,6 +104,8 @@ do --debug) NXT_DEBUG=YES ;; + --hardening=*) NXT_HARDENING="$value" ;; + --no-ipv6) NXT_INET6=NO ;; --no-unix-sockets) NXT_UNIX_DOMAIN=NO ;; diff --git a/auto/sources b/auto/sources index f1ea56685..8a66fb270 100644 --- a/auto/sources +++ b/auto/sources @@ -156,6 +156,8 @@ NXT_TEST_BUILD_SRCS="src/nxt_test_build.c" NXT_TEST_DEPS="src/test/nxt_tests.h \ src/test/nxt_rbtree1.h \ + src/nxt_checked.h \ + src/nxt_span.h \ " NXT_TEST_SRCS=" \ @@ -204,6 +206,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_fd_test.c \ src/test/nxt_port_rpc_fd_test.c \ src/test/nxt_port_queued_fd_test.c \ + src/test/nxt_checked_test.c \ src/test/nxt_conn_close_test.c \ src/test/nxt_router_prepare_msg_test.c \ " diff --git a/auto/summary b/auto/summary index 4a0888a62..2b05d788b 100644 --- a/auto/summary +++ b/auto/summary @@ -39,6 +39,7 @@ Unit configuration summary: cgroupv2: .................. $NXT_HAVE_CGROUP debug logging: ............. $NXT_DEBUG + hardening: ................. $NXT_HARDENING fuzz engine: ............... "$NXT_FUZZ" diff --git a/configure b/configure index 44eb5d8f6..e844a9006 100755 --- a/configure +++ b/configure @@ -74,6 +74,7 @@ mkdir -p $NXT_BUILD_DIR/var/log/unit > $NXT_AUTO_CONFIG_H . auto/cc/test +. auto/cc/hardening cat << END >> $NXT_AUTO_CONFIG_H diff --git a/docs/changes.xml b/docs/changes.xml index e4aa2d400..7177a5408 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,15 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +the --hardening=[off|default|strict] configure option adds compiler and +linker hardening flags. Each flag is probed, and a compiler that does not +support a flag only loses that flag. The default is "off", so a build +without the option does not change. + + + "PUT /certificates/<name>" replaces an existing certificate bundle in diff --git a/src/nxt_checked.h b/src/nxt_checked.h new file mode 100644 index 000000000..45d81ed39 --- /dev/null +++ b/src/nxt_checked.h @@ -0,0 +1,31 @@ + +/* + * Copyright (C) FreeUnit Community + */ + +#ifndef _NXT_CHECKED_H_INCLUDED_ +#define _NXT_CHECKED_H_INCLUDED_ + + +/* + * Checked arithmetic on the compiler's __builtin_*_overflow() (GCC and + * Clang). Each function returns 0 and stores the result in "out". When + * the operation overflows, the function returns 1. Then "out" holds the + * wrapped result. Callers must not use that value. + */ + +nxt_inline int +nxt_size_add(size_t a, size_t b, size_t *out) +{ + return __builtin_add_overflow(a, b, out); +} + + +nxt_inline int +nxt_size_mul(size_t a, size_t b, size_t *out) +{ + return __builtin_mul_overflow(a, b, out); +} + + +#endif /* _NXT_CHECKED_H_INCLUDED_ */ diff --git a/src/nxt_clang.h b/src/nxt_clang.h index 2453bff31..363ce25ea 100644 --- a/src/nxt_clang.h +++ b/src/nxt_clang.h @@ -12,6 +12,19 @@ #define nxt_noinline __attribute__((noinline)) #define nxt_cdecl +/* + * An intended switch fall-through. It is a statement and not a comment. + * Thus -Wimplicit-fallthrough accepts it under both GCC and Clang. That + * warning is part of --hardening=strict, see auto/cc/hardening. A "Fall + * through." comment alone satisfies the default heuristic of GCC. It does + * not satisfy Clang. + */ +#if (NXT_HAVE_GCC_ATTRIBUTE_FALLTHROUGH) +#define nxt_fallthrough __attribute__((__fallthrough__)) +#else +#define nxt_fallthrough +#endif + /* * nxt_static_assert() fails the compilation when the condition does not diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index aa783d01a..713dec0cc 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -2982,7 +2982,7 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) return; } - /* Fall through. */ + nxt_fallthrough; default: case NXT_ERROR: diff --git a/src/nxt_http_parse.c b/src/nxt_http_parse.c index 224ca3f7c..61f1367c3 100644 --- a/src/nxt_http_parse.c +++ b/src/nxt_http_parse.c @@ -358,17 +358,17 @@ nxt_http_parse_request_line(nxt_http_request_parse_t *rp, u_char **pos, if (p[7] < '0' || p[7] > '9') { break; } - /* Fall through. */ + nxt_fallthrough; case 7: if (p[6] != '.') { break; } - /* Fall through. */ + nxt_fallthrough; case 6: if (p[5] < '0' || p[5] > '9') { break; } - /* Fall through. */ + nxt_fallthrough; default: return NXT_AGAIN; } @@ -743,13 +743,13 @@ nxt_http_lookup_field_end(u_char *p, const u_char *end) switch (end - p) { case 3: nxt_field_end_test_char(*p); p++; - /* Fall through. */ + nxt_fallthrough; case 2: nxt_field_end_test_char(*p); p++; - /* Fall through. */ + nxt_fallthrough; case 1: nxt_field_end_test_char(*p); p++; - /* Fall through. */ + nxt_fallthrough; case 0: break; default: diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 6954ad834..b0dacf7dc 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -1721,7 +1721,7 @@ nxt_http_static_extract_extension(nxt_str_t *path, nxt_str_t *exten) switch (ch) { case '/': p++; - /* Fall through. */ + nxt_fallthrough; case '.': goto extension; } diff --git a/src/nxt_murmur_hash.c b/src/nxt_murmur_hash.c index 56bf2d4b6..434b2a0a8 100644 --- a/src/nxt_murmur_hash.c +++ b/src/nxt_murmur_hash.c @@ -37,10 +37,10 @@ nxt_murmur_hash2(const void *data, size_t len) switch (len) { case 3: h ^= p[2] << 16; - /* Fall through. */ + nxt_fallthrough; case 2: h ^= p[1] << 8; - /* Fall through. */ + nxt_fallthrough; case 1: h ^= p[0]; h *= m; diff --git a/src/nxt_router.c b/src/nxt_router.c index 1cb545352..95d05ffa2 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -3604,7 +3604,7 @@ nxt_router_apps_hash_add(nxt_router_conf_t *rtcf, nxt_app_t *app) case NXT_DECLINED: nxt_thread_log_alert("router app hash adding failed: " "\"%V\" is already in hash", &lhq.key); - /* Fall through. */ + nxt_fallthrough; default: return NXT_ERROR; } diff --git a/src/nxt_span.h b/src/nxt_span.h new file mode 100644 index 000000000..8000d272d --- /dev/null +++ b/src/nxt_span.h @@ -0,0 +1,93 @@ + +/* + * Copyright (C) FreeUnit Community + */ + +#ifndef _NXT_SPAN_H_INCLUDED_ +#define _NXT_SPAN_H_INCLUDED_ + +#include + + +/* + * nxt_span_t is a bounded read cursor over a buffer. "pos" is the next + * byte to consume. "end" is one past the last valid byte. It is for + * parsers that walk untrusted, length-prefixed input, such as shared + * memory port messages and libunit request buffers. Such a parser must + * check each advance against the remaining bytes. It must not trust a + * length field that comes from the peer. + * + * A short read means that not enough bytes are left. A partial tail means + * that the span ends in the middle of a whole record. Both are reported + * as a failure. nxt_span_take() never returns a pointer that reads past + * "end". + */ + +typedef struct { + const u_char *pos; + const u_char *end; +} nxt_span_t; + + +nxt_inline void +nxt_span_init(nxt_span_t *span, const u_char *start, const u_char *end) +{ + span->pos = start; + span->end = end; +} + + +nxt_inline size_t +nxt_span_len(const nxt_span_t *span) +{ + return (size_t) (span->end - span->pos); +} + + +/* + * Takes "size" bytes off the front of the span. Returns a pointer to them + * in "out". Returns 1 when fewer than "size" bytes remain. This covers a + * short buffer. It also covers a partial tail that is left after whole + * records were taken. On failure, "span" and "out" do not change. + */ + +nxt_inline int +nxt_span_take(nxt_span_t *span, size_t size, const u_char **out) +{ + size_t avail; + + avail = nxt_span_len(span); + + if (nxt_slow_path(size > avail)) { + return 1; + } + + *out = span->pos; + span->pos += size; + + return 0; +} + + +/* + * Copies "size" bytes off the front of the span into "dst". It fails in + * the same way as nxt_span_take(). On a short or partial tail, the span + * and "dst" do not change. + */ + +nxt_inline int +nxt_span_copy(nxt_span_t *span, void *dst, size_t size) +{ + const u_char *src; + + if (nxt_span_take(span, size, &src) != 0) { + return 1; + } + + nxt_memcpy(dst, src, size); + + return 0; +} + + +#endif /* _NXT_SPAN_H_INCLUDED_ */ diff --git a/src/nxt_string.c b/src/nxt_string.c index 551ba72b9..3f390d6ff 100644 --- a/src/nxt_string.c +++ b/src/nxt_string.c @@ -412,7 +412,7 @@ nxt_strverscmp(const u_char *s1, const u_char *s2) c1 = *s1++; c2 = *s2++; - /* Fall through. */ + nxt_fallthrough; case st_num: @@ -441,7 +441,7 @@ nxt_strverscmp(const u_char *s1, const u_char *s2) return 1; } - /* Fall through. */ + nxt_fallthrough; case st_frac: default: diff --git a/src/nxt_timer.c b/src/nxt_timer.c index cb94b77c8..47f61511d 100644 --- a/src/nxt_timer.c +++ b/src/nxt_timer.c @@ -192,7 +192,7 @@ nxt_timer_changes_commit(nxt_event_engine_t *engine) break; } - /* Fall through. */ + nxt_fallthrough; case NXT_TIMER_DELETE: nxt_debug(timer->task, "timer rbtree delete: %M±%d", diff --git a/src/test/nxt_checked_test.c b/src/test/nxt_checked_test.c new file mode 100644 index 000000000..92ea75f3f --- /dev/null +++ b/src/test/nxt_checked_test.c @@ -0,0 +1,57 @@ +/* + * Copyright (C) FreeUnit Community + */ + +#include +#include +#include +#include "nxt_tests.h" + + +nxt_int_t +nxt_checked_test(nxt_thread_t *thr) +{ + size_t r, extra; + u_char dst[4]; + nxt_span_t span; + const u_char *p; + + static const u_char buf[11] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + + NXT_TEST_CHECK(thr->log, nxt_size_add(2, 3, &r) == 0 && r == 5, + "nxt_size_add(2, 3) failed"); + NXT_TEST_CHECK(thr->log, nxt_size_mul(6, 7, &r) == 0 && r == 42, + "nxt_size_mul(6, 7) failed"); + NXT_TEST_CHECK(thr->log, nxt_size_add(SIZE_MAX, 1, &r) != 0, + "nxt_size_add(SIZE_MAX, 1) did not overflow"); + NXT_TEST_CHECK(thr->log, nxt_size_mul(SIZE_MAX, 2, &r) != 0, + "nxt_size_mul(SIZE_MAX, 2) did not overflow"); + + /* Two whole 4-byte records, then a partial tail of 0..3 bytes. */ + + for (extra = 0; extra <= 3; extra++) { + nxt_span_init(&span, buf, buf + 8 + extra); + + NXT_TEST_CHECK(thr->log, nxt_span_take(&span, 4, &p) == 0 && p == buf + && nxt_span_take(&span, 4, &p) == 0 && p == buf + 4 + && nxt_span_len(&span) == extra, + "nxt_span_take() rejected a whole record"); + NXT_TEST_CHECK(thr->log, nxt_span_take(&span, 4, &p) != 0, + "nxt_span_take() accepted a %uz-byte tail", extra); + } + + nxt_span_init(&span, buf, buf + 6); + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 4) == 0 + && memcmp(dst, buf, 4) == 0, + "nxt_span_copy() failed on a whole record"); + + dst[0] = 0xAA; + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 4) != 0 + && dst[0] == 0xAA, + "nxt_span_copy() accepted or wrote a partial tail"); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nxt_checked test passed"); + return NXT_OK; +} diff --git a/src/test/nxt_lvlhsh_test.c b/src/test/nxt_lvlhsh_test.c index baa6d0e1e..4db4653dd 100644 --- a/src/test/nxt_lvlhsh_test.c +++ b/src/test/nxt_lvlhsh_test.c @@ -57,7 +57,7 @@ nxt_lvlhsh_test_add(nxt_lvlhsh_t *lh, const nxt_lvlhsh_proto_t *proto, case NXT_DECLINED: nxt_thread_log_alert("lvlhsh test failed: " "key %p is already in hash", key); - /* Fall through. */ + nxt_fallthrough; default: return NXT_ERROR; } diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 896d97915..8e29eeecc 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -27,6 +27,44 @@ nxt_test_fd_is_open(nxt_fd_t fd) } +/* + * Runs fn(data) in a child process. Returns the exit status of the child, + * or -1 when the child did not exit. + */ + +int +nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), + void *data) +{ + int status; + pid_t child; + + child = fork(); + + if (child == 0) { + _exit(fn(data)); + } + + if (child == -1 || waitpid(child, &status, 0) != child) { + nxt_log_alert(thr->log, "%s: fork() failed %E", name, nxt_errno); + return -1; + } + + if (!WIFEXITED(status)) { + nxt_log_alert(thr->log, "%s: child killed by signal %d", name, + WTERMSIG(status)); + return -1; + } + + return WEXITSTATUS(status); +} + + +static nxt_int_t (*const nxt_security_tests[])(nxt_thread_t *) = { + nxt_checked_test, +}; + + /* The function is defined here to prevent inline optimizations. */ static nxt_bool_t nxt_msec_less(nxt_msec_t first, nxt_msec_t second) @@ -38,6 +76,7 @@ nxt_msec_less(nxt_msec_t first, nxt_msec_t second) int nxt_cdecl main(int argc, char **argv) { + nxt_uint_t i; nxt_task_t task; nxt_thread_t *thr; @@ -302,6 +341,12 @@ main(int argc, char **argv) return 1; } + for (i = 0; i < nxt_nitems(nxt_security_tests); i++) { + if (nxt_security_tests[i](thr) != NXT_OK) { + return 1; + } + } + if (nxt_conn_close_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 09ae15234..0197eee4c 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -97,11 +97,23 @@ nxt_int_t nxt_port_ctrunc_test(nxt_thread_t *thr); nxt_int_t nxt_port_fd_test(nxt_thread_t *thr); nxt_int_t nxt_port_rpc_fd_test(nxt_thread_t *thr); nxt_int_t nxt_port_queued_fd_test(nxt_thread_t *thr); +nxt_int_t nxt_checked_test(nxt_thread_t *thr); nxt_int_t nxt_conn_close_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); nxt_bool_t nxt_test_fd_is_open(nxt_fd_t fd); +int nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), + void *data); + + +#define NXT_TEST_CHECK(log, cond, ...) \ + do { \ + if (!(cond)) { \ + nxt_log_alert(log, __VA_ARGS__); \ + return NXT_ERROR; \ + } \ + } while (0) #endif /* _NXT_TESTS_H_INCLUDED_ */ From 35d768429e566839547fce919416c62383f8e6fd Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 00:06:36 +0200 Subject: [PATCH 291/391] http: stop serving a range to a client that refused identity A byte range is served as identity. A request with both "Range" and "Accept-Encoding: identity;q=0" was therefore answered with a 206 of the file's own bytes, exactly what the client said it would not take. Reproduced with gzip configured for text/css: Accept-Encoding: gzip, identity;q=0 Range: bytes=0-9 -> 206, no Content-Encoding, body "body{color" Such a request can still be served, because it named a coding Unit has. The Range is ignored, not the request, and the full 200 is sent in that coding. Ignoring a Range is already how this function treats a malformed one, a multi-range one and an If-Range mismatch (RFC 9110 Sect. 14.2). A 406 is the other defensible reading; it refuses a request that can be satisfied, and nginx never reaches the question because it serves the range as identity. The refusal has to be carried, not re-derived. select_compressor() collapsed it into -1, which only means "nothing acceptable"; a request that refuses identity while accepting gzip selects gzip and never reached that return. It now reports identity_refused separately. Two findings from review are folded in. An explicitly named coding outranks the wildcard: "*" was looked up as identity, so "gzip, identity;q=0.5, *;q=0" cleared the flag on the wildcard and nothing restored it on the explicit token; with the range check this made the wildcard veto a range the client had allowed (200 whole file instead of 206). What the wildcard said and what an explicit token said are now tracked apart, and the explicit one wins. And content codings are case-insensitive tokens (Sect. 8.4.1), but the lookup was case-sensitive, so "Identity;q=0" matched nothing and the refusal was dropped; nxt_strcasestr_eq() now, which also makes "GZIP" select gzip. The test decompresses the body and compares it to the file instead of trusting Content-Encoding. A plain "gzip" range keeps its 206 of identity bytes, and "identity;q=0, *;q=0" stays 406 with or without a Range. Closes: https://github.com/freeunitorg/freeunit/issues/355 Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Fable 5.1 --- CHANGES | 10 ++++ docs/changes.xml | 6 +++ src/nxt_http_compression.c | 62 +++++++++++++++++++-- src/nxt_http_compression.h | 1 + src/nxt_http_static.c | 23 ++++++++ test/test_static_compression.py | 96 +++++++++++++++++++++++++++++++++ 6 files changed, 193 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 205224c5d..1d91d6f7c 100644 --- a/CHANGES +++ b/CHANGES @@ -493,6 +493,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 than that is still accepted when nothing else is waiting. A fragmented message also left a count behind that lowered the limit applied to the next message. + *) Bugfix: a byte-range request from a client that refused the identity + coding is no longer answered with identity bytes. A range is served + as identity, so "Range" together with "Accept-Encoding: + identity;q=0" returned a "206 Partial Content" carrying exactly the + bytes the client said it would not take. The range is now ignored + and the full "200 OK" is sent in a coding the client does accept, as + already happens for a malformed or multi-range request. A request + that accepts no coding at all is still answered with "406 Not + Acceptable", and a range from a client that does accept identity is + still a 206. *) Change: unitd, libunit.a and the language modules no longer contain diff --git a/docs/changes.xml b/docs/changes.xml index 7177a5408..c468c5a16 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -651,6 +651,12 @@ connection to 10 MB of payload that the application has not received yet receive()); a single frame larger than that is still accepted when nothing else is waiting. A fragmented message also left a count behind that lowered the limit applied to the next message. +a byte-range request from a client that refused the identity coding is no +longer answered with identity bytes. A range is served as identity, so +Range together with "Accept-Encoding: identity;q=0" returned a 206 carrying +exactly the bytes the client said it would not take; the range is now +ignored and the full 200 is sent in a coding the client accepts. A request +that accepts no coding at all is still 406. diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index d5a51b82a..2c1ba1a48 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -85,6 +85,14 @@ struct nxt_http_comp_ctx_s { */ nxt_int_t sel_idx; + /* + * The client sent "identity;q=0": it will not take the file's own bytes. + * Recorded separately from sel_idx because a request that refuses + * identity and accepts gzip selects gzip and is perfectly serveable -- + * until a Range enters, which is served as identity. + */ + bool identity_refused; + nxt_off_t resp_clen; nxt_off_t clen_sent; @@ -439,6 +447,15 @@ nxt_http_comp_wants_compression(void) } +bool +nxt_http_comp_identity_refused(void) +{ + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + + return ctx->identity_refused; +} + + static nxt_uint_t nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, const nxt_str_t *token) @@ -447,8 +464,15 @@ nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, return NXT_HTTP_COMP_SCHEME_IDENTITY; } + /* + * RFC 9110 Sect. 8.4.1: a content coding is a token, and tokens are + * compared case-insensitively. "Identity;q=0" and "GZIP" are as valid + * as the lowercase spellings, and a case-sensitive compare silently + * ignores them -- which, for identity, means missing a refusal. + */ + for (nxt_uint_t i = 0; i < conf->nr_enabled; i++) { - if (nxt_strstr_eq(token, &conf->enabled[i].type->token)) { + if (nxt_strcasestr_eq(token, &conf->enabled[i].type->token)) { return i; } } @@ -475,13 +499,26 @@ nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, */ static nxt_int_t nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, - nxt_http_request_t *r, const nxt_str_t *token) + nxt_http_request_t *r, const nxt_str_t *token, + bool *identity_refused) { + /* + * "identity_allowed" carries what the wildcard said; "identity_named" + * and "identity_named_ok" carry what an explicit "identity" token said. + * They are kept apart because the explicit one wins: in + * "gzip, identity;q=0.5, *;q=0" the client refused everything it did not + * name and then named identity as acceptable, so identity is acceptable. + * Collapsing the two lets the wildcard veto a coding the client allowed. + */ bool identity_allowed = true; + bool identity_named = false; + bool identity_named_ok = false; char *str, *tkn, *tail, *cur; double weight = 0.0; nxt_int_t idx = NXT_HTTP_COMP_SCHEME_IDENTITY; + *identity_refused = false; + str = nxt_str_cstrz(r->mem_pool, token); if (str == NULL) { return NXT_HTTP_COMP_SCHEME_IDENTITY; @@ -529,8 +566,14 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, scheme = conf->enabled[ecidx].type->scheme; - if (qval == 0.0 && scheme == NXT_HTTP_COMP_SCHEME_IDENTITY) { - identity_allowed = false; + if (scheme == NXT_HTTP_COMP_SCHEME_IDENTITY) { + if (enc.length == 1 && enc.start[0] == '*') { + identity_allowed = (qval != 0.0); + + } else { + identity_named = true; + identity_named_ok = (qval != 0.0); + } } if (qval == 0.0 || qval < weight) { @@ -541,6 +584,12 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, weight = qval; } + if (identity_named) { + identity_allowed = identity_named_ok; + } + + *identity_refused = !identity_allowed; + if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY && !identity_allowed) { return -1; } @@ -828,6 +877,7 @@ nxt_http_comp_merge_vary(nxt_http_request_t *r) nxt_int_t nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) { + bool identity_refused; nxt_int_t ret, idx; nxt_str_t accept_encoding, mime_type = {}; nxt_router_conf_t *rtcf; @@ -899,12 +949,14 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_ERROR; } - idx = nxt_http_comp_select_compressor(conf, r, &accept_encoding); + idx = nxt_http_comp_select_compressor(conf, r, &accept_encoding, + &identity_refused); if (idx == -1) { return NXT_HTTP_NOT_ACCEPTABLE; } ctx->sel_idx = idx; + ctx->identity_refused = identity_refused; return NXT_OK; } diff --git a/src/nxt_http_compression.h b/src/nxt_http_compression.h index 2638ed97d..c172f2ed3 100644 --- a/src/nxt_http_compression.h +++ b/src/nxt_http_compression.h @@ -96,6 +96,7 @@ extern nxt_int_t nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, nxt_file_t **f, nxt_file_info_t *fi, size_t static_buf_len, size_t *out_total); extern bool nxt_http_comp_wants_compression(void); +extern bool nxt_http_comp_identity_refused(void); extern bool nxt_http_comp_compressor_is_valid(const nxt_str_t *token); extern nxt_int_t nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r); diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index b0dacf7dc..7b8cf5c1f 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -921,6 +921,29 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, is_range = (rstatus == NXT_HTTP_PARTIAL_CONTENT); + /* + * A range is served as identity (see the comment on skipping + * compression below), so a client that sent "identity;q=0" must not + * be given one: it asked not to receive the file's own bytes, and a + * 206 hands it exactly those. Such a request is still serveable -- + * it named a coding Unit has -- so drop the Range rather than the + * request, and answer the full 200 in the coding it did accept. + * + * Ignoring a Range is already how this function answers a malformed + * one, a multi-range one and an If-Range mismatch (Sect. 14.2 lets a + * server ignore Range), so this needs no new shape of response. A + * 406 would be the other reading, but it refuses a request that can + * be satisfied, and a client asking for bytes 0-9 of a small file is + * better served the file than an error. + * + * Not reached when nothing is acceptable: that is already 406, from + * nxt_http_comp_check_acceptable() above. + */ + + if (is_range && nxt_http_comp_identity_refused()) { + is_range = 0; + } + if (is_range) { r->status = NXT_HTTP_PARTIAL_CONTENT; r->resp.content_length_n = range_end - range_start + 1; diff --git a/test/test_static_compression.py b/test/test_static_compression.py index f143bf048..3144f0d48 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -1,3 +1,4 @@ +import gzip from pathlib import Path import pytest @@ -334,3 +335,98 @@ def test_static_compression_vary_merge_identity(temp_dir, configured, expected): # emitting "Vary: Origin" takes that path, and nothing here exercises it -- # including the merge's own case-insensitive field-name match. Covering it # needs a language-module test, not a static one. + + +def _raw_get(**headers): + # Raw bytes: the body has to be decompressed, so it must not be decoded. + raw = client.get( + url='/big.css', + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + encoding='latin-1', + read_buffer_size=1024 * 1024, + raw_resp=True, + ) + head, _, body = raw.partition('\r\n\r\n') + lines = head.split('\r\n') + status = int(lines[0].split(' ')[1]) + hdrs = dict(line.split(': ', 1) for line in lines[1:]) + body = body.encode('latin-1') + + if hdrs.get('Transfer-Encoding') == 'chunked': + body = client._parse_chunked_body(body) + + return status, hdrs, body + + +def test_static_compression_range_identity_refused(temp_dir): + # A Range is served as identity -- coding a byte slice would compress the + # wrong bytes -- so a client that sent "identity;q=0" must not be given + # one: it asked not to receive the file's own bytes and a 206 is exactly + # those. The request is still serveable, because it named a coding Unit + # has, so the Range is dropped and the full 200 is sent in that coding. + # + # Without the fix this is a 206 carrying identity bytes to a client that + # refused identity, which is what #355 reports. + data = Path(f'{temp_dir}/assets/big.css').read_bytes() + + status, headers, body = _raw_get( + **{'Accept-Encoding': 'gzip, identity;q=0', 'Range': 'bytes=0-9'} + ) + + assert status == 200, 'the Range is dropped, not the request' + assert headers.get('Content-Encoding') == 'gzip', 'served as gzip' + assert 'Content-Range' not in headers, 'no Content-Range on the full 200' + assert gzip.decompress(body) == data, 'the whole file, correctly coded' + + +def test_static_compression_range_identity_refused_guards(temp_dir): + # The cases either side of it, which must not move. + size = Path(f'{temp_dir}/assets/big.css').stat().st_size + + # Identity acceptable: a Range is still a 206 of identity bytes, which is + # deliberate and is what every other server does. + status, headers, body = _raw_get( + **{'Accept-Encoding': 'gzip', 'Range': 'bytes=0-9'} + ) + assert status == 206, 'gzip alone still gets its partial content' + assert 'Content-Encoding' not in headers, 'a 206 carries no coding' + assert headers['Content-Range'] == f'bytes 0-9/{size}' + assert body == b'body{color' + + # Nothing acceptable at all is 406, whether or not a Range is present. + # That outranks the Range and is unchanged by this fix. + for extra in ({}, {'Range': 'bytes=0-9'}): + status, _, _ = _raw_get( + **{'Accept-Encoding': 'identity;q=0, *;q=0', **extra} + ) + assert status == 406, 'unacceptable outranks any Range' + + # Refusing identity without a Range was already correct: full gzip 200. + status, headers, _ = _raw_get(**{'Accept-Encoding': 'gzip, identity;q=0'}) + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip' + + # An explicitly named identity outranks the wildcard. The client refused + # everything it did not name and then named identity as acceptable, so a + # 206 of identity bytes is exactly what it asked for. Reading the + # wildcard as a veto here drops a range the client could take. + status, headers, body = _raw_get( + **{ + 'Accept-Encoding': 'gzip, identity;q=0.5, *;q=0', + 'Range': 'bytes=0-9', + } + ) + assert status == 206, 'an explicit identity;q>0 keeps its range' + assert 'Content-Encoding' not in headers + assert headers['Content-Range'] == f'bytes 0-9/{size}' + assert body == b'body{color' + + # A content coding is a token and tokens are case-insensitive + # (Sect. 8.4.1), so "Identity;q=0" refuses identity just as "identity;q=0" + # does. A case-sensitive compare drops the refusal on the floor and + # serves the 206 this whole test exists to prevent. + status, headers, _ = _raw_get( + **{'Accept-Encoding': 'gzip, Identity;q=0', 'Range': 'bytes=0-9'} + ) + assert status == 200, 'a mixed-case identity token still refuses' + assert headers.get('Content-Encoding') == 'gzip' From 6240db2356a8b4a75fc089610750d18534e53f53 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 00:06:36 +0200 Subject: [PATCH 292/391] http: parse Accept-Encoding by its grammar; drop a Range wholly or not Five parser defects, each reproduced before the fix. Every one of them let a refusal of identity go unread, so a client that had refused the file's own bytes got a 206 of them. The weight parameter is "OWS ';' OWS ('q' / 'Q') '=' qvalue" (Sect. 12.4.2), but the parse was strstr(tkn, ";q="), so "identity;Q=0" carried no weight Unit could see. OWS is SP or HTAB (Sect. 5.6.3), but the cleanup pass removed only the space, so "identity;q=0" and "identity;q=0" read as unknown codings. "*" is a tchar, so "*foo" is a legal coding name Unit does not have, but the wildcard was matched on the first byte alone; the wildcard is now the whole one-character token. A second Accept-Encoding field was never read. Sect. 5.3 makes several lines one comma-joined value, but nxt_http_var_header() answers with the first matching field, so "Accept-Encoding: gzip" followed by "Accept-Encoding: identity;q=0" reached the parser as "gzip" alone, and in the other order the gzip was never seen and the request drew a 406. The fields are now walked and joined; a single field, which is every ordinary request, still points at the field and copies nothing. The qvalue was read with strtod(qptr + 3, NULL) and no look at where the conversion stopped. strtod() returns 0 when it consumes nothing, and a zero weight is an explicit refusal, so "q=", "q=abc", "q=0x0" and "q=0e0" each answered 406 to a client that had refused nothing, and "q=nan" compared false against both bounds and against the running best, so "gzip;q=nan" was selected. The qvalue shape is checked before converting, and an element whose weight does not parse is ignored, like an element naming an unknown coding. Ignoring the Range was not all or nothing. A satisfiable range from a client that refused identity was dropped, but an unsatisfiable one still answered 416 with "Content-Range: bytes */7500", the size of the very representation the client had refused. The refusal is now tested before the 416 branch. Each defect has its own test. The changelog carries the malformed-weight entry, since that one changed an answer from 406 to a served response. Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Fable 5.1 --- CHANGES | 24 +++- docs/changes.xml | 20 ++- src/nxt_http_compression.c | 238 ++++++++++++++++++++++++++------ src/nxt_http_static.c | 51 ++++--- test/test_static_compression.py | 104 ++++++++++++++ 5 files changed, 375 insertions(+), 62 deletions(-) diff --git a/CHANGES b/CHANGES index 1d91d6f7c..adf6f1bd4 100644 --- a/CHANGES +++ b/CHANGES @@ -498,11 +498,25 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 as identity, so "Range" together with "Accept-Encoding: identity;q=0" returned a "206 Partial Content" carrying exactly the bytes the client said it would not take. The range is now ignored - and the full "200 OK" is sent in a coding the client does accept, as - already happens for a malformed or multi-range request. A request - that accepts no coding at all is still answered with "406 Not - Acceptable", and a range from a client that does accept identity is - still a 206. + and the full "200 OK" is sent in a coding the client does accept. + Ignoring the range is all or nothing, so an unsatisfiable range from + such a client is no longer answered with "416" either. A request + that accepts no coding at all is still "406 Not Acceptable", and a + range from a client that does accept identity is still a 206. + Several "Accept-Encoding" lines in one request are now read as the + single comma-joined value RFC 9110 defines them to be; previously + only the first was consulted. + + *) Bugfix: a malformed weight in "Accept-Encoding" is no longer read as a + refusal. The qvalue was converted with strtod(), which takes no digits + at all from "identity;q=" and from "identity;q=abc" and reports zero, + so either one answered "406 Not Acceptable" to a client that had + refused nothing; "q=0x0" and "q=0e0" did the same. An element whose + weight does not match the grammar of RFC 9110 Sect. 12.4.2 is now + ignored, as an element naming an unknown coding already was. This + also removes "q=nan", which passed the 0-to-1 range check -- a NaN + compares false against both bounds -- and then outranked every real + weight. *) Change: unitd, libunit.a and the language modules no longer contain diff --git a/docs/changes.xml b/docs/changes.xml index c468c5a16..8fe82e3ca 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -655,8 +655,24 @@ a byte-range request from a client that refused the identity coding is no longer answered with identity bytes. A range is served as identity, so Range together with "Accept-Encoding: identity;q=0" returned a 206 carrying exactly the bytes the client said it would not take; the range is now -ignored and the full 200 is sent in a coding the client accepts. A request -that accepts no coding at all is still 406. +ignored and the full 200 is sent in a coding the client accepts. An +unsatisfiable range from such a client is no longer answered with 416 either. +A request that accepts no coding at all is still 406. Several Accept-Encoding +lines in one request are now read as one comma-joined value; previously only the +first was consulted. + + + + + +a malformed weight in Accept-Encoding is no longer read as a refusal. The +qvalue was converted with strtod(), which takes no digits at all from +"identity;q=" and from "identity;q=abc" and reports zero, so either one +answered 406 to a client that had refused nothing; "q=0x0" and "q=0e0" did the +same. An element whose weight does not match the grammar of RFC 9110 +Sect. 12.4.2 is now ignored, as an element naming an unknown coding already +was. This also removes "q=nan", which passed the 0-to-1 range check and then +outranked every real weight. diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 2c1ba1a48..a69771111 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -108,7 +108,6 @@ struct nxt_http_comp_ctx_s { * router on the next request (#167). */ struct nxt_http_comp_conf_s { - nxt_tstr_t *accept_encoding_query; nxt_http_route_rule_t *mime_types_rule; nxt_http_comp_compressor_t *enabled; nxt_uint_t nr_enabled; @@ -456,11 +455,93 @@ nxt_http_comp_identity_refused(void) } +/* + * Finds the ";q=" weight parameter in one Accept-Encoding element. + * + * RFC 9110 Sect. 12.4.2 spells the parameter "weight = OWS ';' OWS ('q' / + * 'Q') '=' qvalue", and an ABNF literal is case-insensitive besides, so + * "identity;Q=0" is as valid as "identity;q=0". A plain strstr() for ";q=" + * misses it, and for identity that means missing a refusal. Spaces are + * already gone by the time this runs. + */ + +static char * +nxt_http_comp_find_weight(char *tkn) +{ + for (char *p = tkn; (p = strchr(p, ';')) != NULL; p++) { + if ((p[1] == 'q' || p[1] == 'Q') && p[2] == '=') { + return p; + } + } + + return NULL; +} + + +/* + * Reads the qvalue that follows a weight's "=". + * + * RFC 9110 Sect. 12.4.2 spells it "( '0' [ '.' 0*3DIGIT ] ) / ( '1' [ '.' + * 0*3( '0' ) ] )": one leading digit, then at most a fraction. strtod() on + * its own is far looser and every way it is looser is a bug here. It takes + * no digits at all from "q=" and from "q=abc" and reports 0, which this + * function's caller reads as a refusal; it reads "q=0x10" as hexadecimal; + * and it turns "q=nan" into a NaN that compares false against both range + * bounds, so the element is kept and then outranks every real weight. + * + * So check the shape first and only then convert. The digit count in the + * fraction is not enforced: rejecting "q=0.0000" would read a client's + * refusal as an acceptance, which is the wrong way to be strict. + * + * An element whose weight does not parse is ignored, exactly like an element + * naming a coding this build does not have. Reading it as q=0 is the other + * defensible answer -- nginx reads a bad quantity as zero -- but a zero is a + * refusal here, so "identity;q=" would answer 406 to a client that refused + * nothing. + */ + +static bool +nxt_http_comp_parse_weight(const char *qvalue, double *qval) +{ + const char *p = qvalue; + + if (*p != '0' && *p != '1') { + return false; + } + + p++; + + if (*p == '.') { + p++; + + while (*p >= '0' && *p <= '9') { + p++; + } + } + + /* Anything may follow the weight, but only as a further parameter. */ + + if (*p != '\0' && *p != ';') { + return false; + } + + *qval = strtod(qvalue, NULL); + + return *qval <= 1.0; +} + + static nxt_uint_t nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, const nxt_str_t *token) { - if (token->start[0] == '*') { + /* + * The wildcard is the whole token, not merely its first character: + * "*" is a tchar, so "*foo" is a legal (and unknown) coding name, and + * matching on the first byte alone made it stand for every coding. + */ + + if (token->length == 1 && token->start[0] == '*') { return NXT_HTTP_COMP_SCHEME_IDENTITY; } @@ -497,6 +578,104 @@ nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, * * 'identity;q=0' seems to basically mean the same thing... */ +/* + * Collects every Accept-Encoding field into one value. + * + * RFC 9110 Sect. 5.3: a field that may carry a comma-separated list can be + * sent as several lines, and a recipient must treat them as one value joined + * by commas. A variable query answers with the first matching field only + * (nxt_http_var_header()), so "Accept-Encoding: gzip" followed by + * "Accept-Encoding: identity;q=0" lost the refusal and the request was served + * the identity bytes it had declined -- and in the other order the gzip it + * would have accepted was never seen, so it drew a 406. + * + * The single-field case, which is every ordinary request, points straight at + * the field and copies nothing. + */ + +static nxt_int_t +nxt_http_comp_accept_encoding(nxt_http_request_t *r, nxt_str_t *value) +{ + u_char *p; + size_t len; + nxt_uint_t n; + nxt_http_field_t *f, *first; + nxt_http_fields_iter_t iter; + + static const nxt_str_t accept_encoding = nxt_string("Accept-Encoding"); + + n = 0; + len = 0; + first = NULL; + + for (f = nxt_http_fields_first(&iter, r->inline_fields, + r->num_inline_fields, r->fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip || f->name_length != accept_encoding.length + || nxt_strncasecmp(f->name, accept_encoding.start, + accept_encoding.length) != 0) + { + continue; + } + + if (n == 0) { + first = f; + + } else { + len += nxt_length(", "); + } + + len += f->value_length; + n++; + } + + if (n == 0) { + nxt_str_null(value); + return NXT_OK; + } + + if (n == 1) { + value->start = first->value; + value->length = first->value_length; + + return NXT_OK; + } + + p = nxt_mp_nget(r->mem_pool, len); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + value->start = p; + value->length = len; + + n = 0; + + for (f = nxt_http_fields_first(&iter, r->inline_fields, + r->num_inline_fields, r->fields); + f != NULL; + f = nxt_http_fields_next(&iter)) + { + if (f->skip || f->name_length != accept_encoding.length + || nxt_strncasecmp(f->name, accept_encoding.start, + accept_encoding.length) != 0) + { + continue; + } + + if (n++ != 0) { + p = nxt_cpymem(p, ", ", nxt_length(", ")); + } + + p = nxt_cpymem(p, f->value, f->value_length); + } + + return NXT_OK; +} + + static nxt_int_t nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, nxt_http_request_t *r, const nxt_str_t *token, @@ -526,11 +705,17 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, cur = tail = str; /* - * To ease parsing the Accept-Encoding header, remove all spaces, - * which hold no semantic meaning. + * To ease parsing the Accept-Encoding header, remove all optional + * whitespace, which holds no semantic meaning. + * + * OWS is SP or HTAB (RFC 9110 Sect. 5.6.3), and it is legal on either + * side of the weight's semicolon. Removing only the space left + * "identity;q=0" unparsed, so the element read as an unknown + * coding and the refusal it carried was lost -- which handed a client + * that refused identity a 206 of exactly those bytes. */ for (; *cur != '\0'; cur++) { - if (*cur == ' ') { + if (*cur == ' ' || *cur == '\t') { continue; } @@ -545,15 +730,9 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, nxt_uint_t ecidx; nxt_http_comp_scheme_t scheme; - qptr = strstr(tkn, ";q="); - if (qptr != NULL) { - nxt_errno = 0; - - qval = strtod(qptr + 3, NULL); - - if (nxt_errno == ERANGE || qval < 0.0 || qval > 1.0) { - continue; - } + qptr = nxt_http_comp_find_weight(tkn); + if (qptr != NULL && !nxt_http_comp_parse_weight(qptr + 3, &qval)) { + continue; } enc.start = (u_char *)tkn; @@ -877,12 +1056,11 @@ nxt_http_comp_merge_vary(nxt_http_request_t *r) nxt_int_t nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) { - bool identity_refused; - nxt_int_t ret, idx; - nxt_str_t accept_encoding, mime_type = {}; - nxt_router_conf_t *rtcf; - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - nxt_http_comp_conf_t *conf = nxt_http_comp_request_conf(r); + bool identity_refused; + nxt_int_t ret, idx; + nxt_str_t accept_encoding, mime_type = {}; + nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_conf_t *conf = nxt_http_comp_request_conf(r); *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1, .sel_idx = -1 }; @@ -920,8 +1098,6 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) } } - rtcf = r->conf->socket_conf->router_conf; - if (nxt_http_comp_is_resp_content_encoded(r)) { return NXT_OK; } @@ -937,14 +1113,7 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_ERROR; } - ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, &r->tstr_cache, - r, r->mem_pool); - if (nxt_slow_path(ret == NXT_ERROR)) { - return NXT_ERROR; - } - - ret = nxt_tstr_query(task, r->tstr_query, conf->accept_encoding_query, - &accept_encoding); + ret = nxt_http_comp_accept_encoding(r, &accept_encoding); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; } @@ -1174,8 +1343,6 @@ nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_value_t *comps, *mimes; nxt_http_comp_conf_t *conf; - static const nxt_str_t accept_enc_str = - nxt_string("$header_accept_encoding"); static const nxt_str_t comps_str = nxt_string("compressors"); static const nxt_str_t mimes_str = nxt_string("types"); @@ -1194,13 +1361,6 @@ nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, } } - conf->accept_encoding_query = - nxt_tstr_compile(rtcf->tstr_state, &accept_enc_str, - NXT_TSTR_STRZ); - if (nxt_slow_path(conf->accept_encoding_query == NULL)) { - return NXT_ERROR; - } - comps = nxt_conf_get_object_member(comp_conf, &comps_str, NULL); if (nxt_slow_path(comps == NULL)) { return NXT_ERROR; diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 7b8cf5c1f..2806c64f8 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -889,6 +889,20 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_file_size(&fi), &range_start, &range_end); + /* + * A range is served as identity, so a client that sent + * "identity;q=0" must not be given one -- see the comment on the + * drop below. Tested here rather than after the 416 branch so that + * ignoring the Range is all or nothing: an unsatisfiable range from + * such a client would otherwise still draw a 416 whose + * "Content-Range: bytes STAR/size" reports the size of exactly the + * representation it refused. + */ + + if (nxt_http_comp_identity_refused()) { + rstatus = NXT_HTTP_OK; + } + if (rstatus == NXT_HTTP_RANGE_NOT_SATISFIABLE) { nxt_file_close(task, f); f = NULL; @@ -922,28 +936,33 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, is_range = (rstatus == NXT_HTTP_PARTIAL_CONTENT); /* - * A range is served as identity (see the comment on skipping - * compression below), so a client that sent "identity;q=0" must not - * be given one: it asked not to receive the file's own bytes, and a - * 206 hands it exactly those. Such a request is still serveable -- - * it named a coding Unit has -- so drop the Range rather than the - * request, and answer the full 200 in the coding it did accept. + * Why the Range is dropped for a client that refused identity. * - * Ignoring a Range is already how this function answers a malformed - * one, a multi-range one and an If-Range mismatch (Sect. 14.2 lets a - * server ignore Range), so this needs no new shape of response. A - * 406 would be the other reading, but it refuses a request that can - * be satisfied, and a client asking for bytes 0-9 of a small file is - * better served the file than an error. + * Not because a coded range is impossible: Sect. 3.2 defines + * representation data as the bytes after content coding and Sect. + * 14.1.2 defines a byte range over that, so bytes 0-9 of the gzip + * representation is well defined -- and Unit codes the whole file + * into a temp file before sending, so it could slice that instead. + * + * Because a coded range is never resumable, and costs a whole file + * to answer. A coded representation carries a weak entity-tag + * (nxt_http_comp_weaken_etag()), and nxt_http_static_range() above + * compares an "If-Range" strongly, so a resumed coded range always + * falls back to the full response -- the one thing a range request + * exists to avoid. Meanwhile Unit compresses the whole file into a + * temp file to hand back ten bytes of it. + * + * So ignore the Range (Sect. 14.2 permits it) and send the full 200 + * in the coding the client did accept. That is already how this + * function answers a malformed range, a multi-range request and an + * If-Range mismatch, so it needs no new shape of response. A 406 + * would be the other reading, but it refuses a request that can be + * satisfied. * * Not reached when nothing is acceptable: that is already 406, from * nxt_http_comp_check_acceptable() above. */ - if (is_range && nxt_http_comp_identity_refused()) { - is_range = 0; - } - if (is_range) { r->status = NXT_HTTP_PARTIAL_CONTENT; r->resp.content_length_n = range_end - range_start + 1; diff --git a/test/test_static_compression.py b/test/test_static_compression.py index 3144f0d48..bfbeda597 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -430,3 +430,107 @@ def test_static_compression_range_identity_refused_guards(temp_dir): ) assert status == 200, 'a mixed-case identity token still refuses' assert headers.get('Content-Encoding') == 'gzip' + + # The weight is "('q' / 'Q') '=' qvalue" (Sect. 12.4.2), and an ABNF + # literal is case-insensitive anyway. A strstr() for ";q=" alone misses + # ";Q=", and for identity a missed weight is a missed refusal -- the + # request gets the 206 of identity bytes it asked not to receive. + status, headers, _ = _raw_get( + **{'Accept-Encoding': 'gzip, identity;Q=0', 'Range': 'bytes=0-9'} + ) + assert status == 200, 'an uppercase Q still refuses' + assert headers.get('Content-Encoding') == 'gzip' + + # "*" is a tchar, so "*foo" is a legal coding name that Unit does not + # have -- not the wildcard. Matching the wildcard on the first byte + # alone made an unknown coding refuse identity and cost the client a + # range it could have taken. + status, headers, body = _raw_get( + **{'Accept-Encoding': 'gzip, *foo;q=0', 'Range': 'bytes=0-9'} + ) + assert status == 206, 'an unknown coding is not the wildcard' + assert 'Content-Encoding' not in headers + assert headers['Content-Range'] == f'bytes 0-9/{size}' + assert body == b'body{color' + + # OWS is SP or HTAB (Sect. 5.6.3) and is legal either side of the + # weight's semicolon. Stripping only the space left the tab forms + # unparsed, so the element read as an unknown coding and took its + # refusal with it. + for spelling in ( + 'gzip, identity; q=0', + 'gzip, identity ;q=0', + 'gzip, identity; q=0', + ): + status, headers, _ = _raw_get( + **{'Accept-Encoding': spelling, 'Range': 'bytes=0-9'} + ) + assert status == 200, f'whitespace in the weight: {spelling!r}' + assert headers.get('Content-Encoding') == 'gzip' + + # Sect. 5.3: a list-valued field may arrive as several lines and must be + # read as one value joined by commas. A variable query answers with the + # first matching field only, so the second line went unread: in this order + # the refusal was lost and the 206 went out anyway, and in the other order + # the gzip the client would have taken was never seen and it drew a 406. + for lines_ae in (['gzip', 'identity;q=0'], ['identity;q=0', 'gzip']): + status, headers, _ = _raw_get( + **{'Accept-Encoding': lines_ae, 'Range': 'bytes=0-9'} + ) + assert status == 200, f'repeated field: {lines_ae!r}' + assert headers.get('Content-Encoding') == 'gzip' + + # Ignoring the Range is all or nothing. An unsatisfiable range from a + # client that refused identity would otherwise draw a 416 whose + # "Content-Range: bytes */size" reports the size of the very + # representation it refused. + status, headers, _ = _raw_get( + **{'Accept-Encoding': 'gzip, identity;q=0', 'Range': 'bytes=99999-'} + ) + assert status == 200, 'unsatisfiable range is ignored too' + assert headers.get('Content-Encoding') == 'gzip' + assert 'Content-Range' not in headers + + # A client that accepts identity still gets its 416. + status, headers, _ = _raw_get( + **{'Accept-Encoding': 'gzip', 'Range': 'bytes=99999-'} + ) + assert status == 416, 'an ordinary unsatisfiable range is still 416' + assert headers['Content-Range'] == f'bytes */{size}' + + +def test_static_compression_malformed_weight(temp_dir): + # A qvalue is "('0' ['.' 0*3DIGIT]) / ('1' ['.' 0*3('0')])" (Sect. + # 12.4.2). strtod() takes no digits at all from "q=" and "q=abc" and + # reports 0, reads "q=0x0" as hexadecimal and "q=0e0" as an exponent, so + # every one of these was read as an explicit refusal and answered 406 to + # a client that refused nothing. + size = Path(f'{temp_dir}/assets/big.css').stat().st_size + + for weight in ('', 'abc', '0x0', '0e0', 'nan', '.5', '2', '1.5'): + status, headers, body = _raw_get( + **{'Accept-Encoding': f'identity;q={weight}'} + ) + assert status == 200, f'a malformed weight is not a refusal: {weight!r}' + assert 'Content-Encoding' not in headers + assert len(body) == size + + # "q=nan" was worse than a refusal on any other coding: a NaN compares + # false against both range bounds, so the element survived the check, and + # false against the running best weight, so it was then selected. + status, headers, body = _raw_get(**{'Accept-Encoding': 'gzip;q=nan'}) + assert status == 200 + assert 'Content-Encoding' not in headers, 'a NaN weight selects nothing' + assert len(body) == size + + # A well-formed zero still refuses, including a fraction longer than the + # grammar's three digits: being strict about the digit count would read a + # refusal as an acceptance, which is the wrong way to be strict. + for weight in ('0', '0.0', '0.000', '0.0000'): + status, _, _ = _raw_get(**{'Accept-Encoding': f'identity;q={weight}'}) + assert status == 406, f'a well-formed zero still refuses: {weight!r}' + + # A parameter after the weight does not make the weight malformed. + status, headers, _ = _raw_get(**{'Accept-Encoding': 'gzip;q=0.5;ext=1'}) + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip', 'q=0.5 then an extension' From 37fe59b9a7cc9688143c2a54b428add5fa3d42b7 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 00:06:36 +0200 Subject: [PATCH 293/391] router: answer 406, not 503, on a negotiation failure Every result of nxt_http_comp_check_acceptable() other than NXT_OK was a generic failure in nxt_router_response_ready_handler(), whose "fail:" path answers 503. A request that refused every coding Unit could apply was therefore reported as the server being unable to serve anybody: with gzip configured for text/html, "Accept-Encoding: identity;q=0" against a PHP response returned 503 where the static path returned 406. The handler now carries the status the check returned, and answers 406. The release below the label is still what the chain needs; only the answer changes. The static path already answered 406 and is untouched. Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Fable 5.1 --- CHANGES | 6 ++++++ docs/changes.xml | 9 +++++++++ src/nxt_router.c | 17 ++++++++++++++++- test/test_php_compression.py | 18 ++++++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index adf6f1bd4..8c827077b 100644 --- a/CHANGES +++ b/CHANGES @@ -518,6 +518,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 compares false against both bounds -- and then outranked every real weight. + *) Bugfix: a negotiation failure on "Accept-Encoding" was answered with + "503 Service Unavailable" instead of "406 Not Acceptable" when the + response came from an application. The router treated every result of + the acceptability check other than success as a server error; only the + static path reported the 406. + *) Change: unitd, libunit.a and the language modules no longer contain the test hooks compiled under "#if (NXT_TESTS)". "./configure diff --git a/docs/changes.xml b/docs/changes.xml index 8fe82e3ca..6031dc3f6 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -676,6 +676,15 @@ outranked every real weight. + + +a negotiation failure on Accept-Encoding was answered with 503 instead of 406 +when the response came from an application. The router treated every result +of the acceptability check other than success as a server error; only the +static path reported the 406. + + + unitd, libunit.a and the language modules no longer contain the test hooks diff --git a/src/nxt_router.c b/src/nxt_router.c index 95d05ffa2..dd6afdfc7 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5350,10 +5350,14 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_t *app_port; nxt_unit_field_t *f; nxt_http_field_t *field; + nxt_http_status_t status; nxt_http_request_t *r; nxt_unit_response_t *resp; nxt_request_rpc_data_t *req_rpc_data; + /* What "fail:" answers with, unless a branch below knows better. */ + status = NXT_HTTP_SERVICE_UNAVAILABLE; + /* * An application response never legitimately carries a descriptor and * this handler does not read msg->fd. The dispatcher closes what a @@ -5553,6 +5557,17 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, */ ret = nxt_http_comp_check_acceptable(task, r); if (ret != NXT_OK) { + /* + * No acceptable representation is a fault of the request, not of + * the server, and it is the one non-NXT_OK result here that is + * not an error: the release below is still what the chain needs, + * but the answer is 406 rather than the 503 "fail:" gives + * everything else. + */ + if (ret == NXT_HTTP_NOT_ACCEPTABLE) { + status = NXT_HTTP_NOT_ACCEPTABLE; + } + goto fail; } @@ -5638,7 +5653,7 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, r->last = last_b; } - nxt_http_request_error(task, r, NXT_HTTP_SERVICE_UNAVAILABLE); + nxt_http_request_error(task, r, status); /* * Complete the buffers adopted from the port above, if any: nobody else diff --git a/test/test_php_compression.py b/test/test_php_compression.py index 3555a5785..171019496 100644 --- a/test/test_php_compression.py +++ b/test/test_php_compression.py @@ -277,3 +277,21 @@ def test_php_compression_shm_not_leaked(): hashlib.sha256(gzip.decompress(body)).hexdigest() == headers['X-Body-Sha256'] ), f'body of request {i}' + + +def test_php_compression_identity_refused(): + # An application response is negotiated like a static one. The client + # refuses identity and names no coding Unit can apply, so no acceptable + # representation is left and the answer is 406. + # + # nxt_router_response_ready_handler() used to read every non-NXT_OK + # result from nxt_http_comp_check_acceptable() as a server error and take + # the generic "fail:" path, which answers 503 -- a negotiation failure + # reported as the server being unable to serve anybody. + client.load('comp_large_body') + configure_compression() + + status, headers, _ = raw_get('/', 'identity;q=0') + + assert status == 406, 'no representation left for the application body' + assert 'Content-Encoding' not in headers From f680aa3b739d7be1830e2eb820a209952df7ba8c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 23:15:27 +0000 Subject: [PATCH 294/391] build: address the review of --hardening Fixes to the --hardening change, in the order of the review: - CI: add the "hardening-strict" job to build-test.yml. It configures --hardening=strict --tests --openssl with gcc and with clang, builds unitd and the C tests, and runs them. It fails when a probed flag is not found. Thus a new warning under -Wformat=2 or -Wvla, a switch fall-through without nxt_fallthrough, or a compiler that drops a flag fails the pull request; - nxt_span_copy() with size 0 returns early. Before, an empty span with a NULL "pos" reached memcpy(dst, NULL, 0), which is undefined. nxt_checked_test.c covers a copy of 0 bytes from an empty span and from a non-empty span; - nxt_checked.h: probe __builtin_add_overflow() in auto/clang (NXT_HAVE_BUILTIN_OVERFLOW). Without it, nxt_size_add() and nxt_size_mul() call the portable nxt_size_add_portable() and nxt_size_mul_portable(). The portable functions are always compiled. The test runs both sets over one table with the SIZE_MAX edges; - nxt_span.h does not include nxt_checked.h any more, since it uses nothing from it. nxt_test_in_child() reports a fork() failure and a waitpid() failure with the right name, and it retries waitpid() on EINTR. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 63 +++++++++++++++++ auto/clang | 18 +++++ src/nxt_checked.h | 58 ++++++++++++++-- src/nxt_span.h | 10 ++- src/test/nxt_checked_test.c | 114 ++++++++++++++++++++++++++++--- src/test/nxt_tests.c | 12 +++- 6 files changed, 257 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6e97e446e..c851f5973 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1006,3 +1006,66 @@ jobs: - name: Run C tests run: sudo ./build/tests + + # --hardening=strict compile gate, with gcc and with clang. auto/cc/test + # already passes -Werror to both, so a new warning from -Wformat=2, -Wvla or + # -Wimplicit-fallthrough (a fall-through without nxt_fallthrough) fails the + # job. The assert step fails when a compiler stops accepting one of the + # probed flags: auto/cc/hardening only drops such a flag, so a plain build + # would pass while testing nothing. Cheap on purpose: a build plus the C + # test suite, no language modules. + hardening-strict: + runs-on: ubuntu-latest + timeout-minutes: 20 + + strategy: + fail-fast: false + matrix: + cc: [gcc, clang] + + steps: + - uses: actions/checkout@v5 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libssl-dev libpcre2-dev clang + + - name: Configure unit (--hardening=strict --tests --openssl) + run: | + ./configure \ + --cc=${{ matrix.cc }} \ + --hardening=strict \ + --tests \ + --openssl 2>&1 | tee configure.log + test "${PIPESTATUS[0]}" -eq 0 + + # Every flag that this job exists for must be found. The FORTIFY probe + # falls back from level 3 to 2, so either level passes. The CET and + # branch-protection flags are not asserted: each applies to one target + # only. + - name: Assert configure found the hardening flags + run: | + grep -q 'hardening: .* strict$' configure.log + for flag in -fstack-protector-strong -fstack-clash-protection \ + -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack \ + -Wformat=2 -Wvla -Wimplicit-fallthrough \ + -ftrivial-auto-var-init=pattern + do + grep -q "checking for $flag \.\.\. found" configure.log \ + || { echo "$flag was not found"; exit 1; } + done + grep -q 'checking for -D_FORTIFY_SOURCE=[23] \.\.\. found' \ + configure.log \ + || { echo "_FORTIFY_SOURCE was not found"; exit 1; } + grep -q 'define NXT_HAVE_BUILTIN_OVERFLOW' \ + build/include/nxt_auto_config.h + + - name: Build unit + run: make -j4 + + - name: Build C tests + run: make -j4 tests + + - name: Run C tests + run: sudo ./build/tests diff --git a/auto/clang b/auto/clang index 87ab8fafc..51698d28c 100644 --- a/auto/clang +++ b/auto/clang @@ -114,6 +114,24 @@ nxt_feature_test="int main(void) { . auto/feature +nxt_feature="GCC __builtin_add_overflow()" +nxt_feature_name=NXT_HAVE_BUILTIN_OVERFLOW +nxt_feature_run= +nxt_feature_incs= +nxt_feature_libs= +nxt_feature_test="#include + + int main(void) { + size_t r; + + if (__builtin_add_overflow((size_t) 2, (size_t) 3, &r) + || __builtin_mul_overflow((size_t) 2, (size_t) 3, &r)) + return 1; + return 0; + }" +. auto/feature + + nxt_feature="GCC __attribute__ visibility" nxt_feature_name=NXT_HAVE_GCC_ATTRIBUTE_VISIBILITY nxt_feature_run= diff --git a/src/nxt_checked.h b/src/nxt_checked.h index 45d81ed39..d9a0c9c14 100644 --- a/src/nxt_checked.h +++ b/src/nxt_checked.h @@ -1,4 +1,3 @@ - /* * Copyright (C) FreeUnit Community */ @@ -8,12 +7,44 @@ /* - * Checked arithmetic on the compiler's __builtin_*_overflow() (GCC and - * Clang). Each function returns 0 and stores the result in "out". When - * the operation overflows, the function returns 1. Then "out" holds the - * wrapped result. Callers must not use that value. + * Checked size_t arithmetic. Each function returns 0 and stores the + * result in "out". When the operation overflows, the function returns 1. + * Then the value of "out" is not defined. Callers must not use it. + * + * With GCC and Clang, nxt_size_add() and nxt_size_mul() use the compiler's + * __builtin_*_overflow() (probed in auto/clang). Other compilers get the + * portable functions. The portable functions are always compiled, so + * that the test suite covers them under every compiler. */ +nxt_inline int +nxt_size_add_portable(size_t a, size_t b, size_t *out) +{ + if (b > SIZE_MAX - a) { + return 1; + } + + *out = a + b; + + return 0; +} + + +nxt_inline int +nxt_size_mul_portable(size_t a, size_t b, size_t *out) +{ + if (a != 0 && b > SIZE_MAX / a) { + return 1; + } + + *out = a * b; + + return 0; +} + + +#if (NXT_HAVE_BUILTIN_OVERFLOW) + nxt_inline int nxt_size_add(size_t a, size_t b, size_t *out) { @@ -27,5 +58,22 @@ nxt_size_mul(size_t a, size_t b, size_t *out) return __builtin_mul_overflow(a, b, out); } +#else + +nxt_inline int +nxt_size_add(size_t a, size_t b, size_t *out) +{ + return nxt_size_add_portable(a, b, out); +} + + +nxt_inline int +nxt_size_mul(size_t a, size_t b, size_t *out) +{ + return nxt_size_mul_portable(a, b, out); +} + +#endif + #endif /* _NXT_CHECKED_H_INCLUDED_ */ diff --git a/src/nxt_span.h b/src/nxt_span.h index 8000d272d..6418e2d14 100644 --- a/src/nxt_span.h +++ b/src/nxt_span.h @@ -6,8 +6,6 @@ #ifndef _NXT_SPAN_H_INCLUDED_ #define _NXT_SPAN_H_INCLUDED_ -#include - /* * nxt_span_t is a bounded read cursor over a buffer. "pos" is the next @@ -72,7 +70,9 @@ nxt_span_take(nxt_span_t *span, size_t size, const u_char **out) /* * Copies "size" bytes off the front of the span into "dst". It fails in * the same way as nxt_span_take(). On a short or partial tail, the span - * and "dst" do not change. + * and "dst" do not change. A copy of 0 bytes always succeeds and does + * not touch "dst". It also does not call memcpy(), because "pos" of an + * empty span may be NULL and memcpy(dst, NULL, 0) is undefined. */ nxt_inline int @@ -80,6 +80,10 @@ nxt_span_copy(nxt_span_t *span, void *dst, size_t size) { const u_char *src; + if (size == 0) { + return 0; + } + if (nxt_span_take(span, size, &src) != 0) { return 1; } diff --git a/src/test/nxt_checked_test.c b/src/test/nxt_checked_test.c index 92ea75f3f..c8aaa3480 100644 --- a/src/test/nxt_checked_test.c +++ b/src/test/nxt_checked_test.c @@ -8,24 +8,100 @@ #include "nxt_tests.h" +typedef struct { + size_t a; + size_t b; + size_t result; + int overflow; +} nxt_checked_case_t; + + +static const nxt_checked_case_t nxt_checked_add_cases[] = { + { 0, 0, 0, 0 }, + { 2, 3, 5, 0 }, + { SIZE_MAX, 0, SIZE_MAX, 0 }, + { 0, SIZE_MAX, SIZE_MAX, 0 }, + { SIZE_MAX - 1, 1, SIZE_MAX, 0 }, + { SIZE_MAX, 1, 0, 1 }, + { 1, SIZE_MAX, 0, 1 }, + { SIZE_MAX, SIZE_MAX, 0, 1 }, +}; + + +static const nxt_checked_case_t nxt_checked_mul_cases[] = { + { 0, 0, 0, 0 }, + { 0, SIZE_MAX, 0, 0 }, + { SIZE_MAX, 0, 0, 0 }, + { 6, 7, 42, 0 }, + { SIZE_MAX, 1, SIZE_MAX, 0 }, + { 1, SIZE_MAX, SIZE_MAX, 0 }, + { SIZE_MAX / 2, 2, SIZE_MAX - 1, 0 }, + { SIZE_MAX / 2 + 1, 2, 0, 1 }, + { SIZE_MAX, 2, 0, 1 }, + { 2, SIZE_MAX, 0, 1 }, + { SIZE_MAX, SIZE_MAX, 0, 1 }, +}; + + +/* + * Runs one add or mul function over its table. The builtin and the + * portable functions must agree on every row, so both are run over the + * same table. + */ + +static nxt_int_t +nxt_checked_run_table(nxt_thread_t *thr, const char *name, + int (*fn)(size_t, size_t, size_t *), const nxt_checked_case_t *cases, + nxt_uint_t n) +{ + size_t r; + nxt_uint_t i; + + for (i = 0; i < n; i++) { + r = 0xAB; + + if (cases[i].overflow) { + NXT_TEST_CHECK(thr->log, fn(cases[i].a, cases[i].b, &r) != 0, + "%s(%uz, %uz) did not overflow", + name, cases[i].a, cases[i].b); + continue; + } + + NXT_TEST_CHECK(thr->log, fn(cases[i].a, cases[i].b, &r) == 0 + && r == cases[i].result, + "%s(%uz, %uz) failed: %uz, expected %uz", + name, cases[i].a, cases[i].b, r, cases[i].result); + } + + return NXT_OK; +} + + nxt_int_t nxt_checked_test(nxt_thread_t *thr) { - size_t r, extra; + size_t extra; u_char dst[4]; nxt_span_t span; const u_char *p; static const u_char buf[11] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; - NXT_TEST_CHECK(thr->log, nxt_size_add(2, 3, &r) == 0 && r == 5, - "nxt_size_add(2, 3) failed"); - NXT_TEST_CHECK(thr->log, nxt_size_mul(6, 7, &r) == 0 && r == 42, - "nxt_size_mul(6, 7) failed"); - NXT_TEST_CHECK(thr->log, nxt_size_add(SIZE_MAX, 1, &r) != 0, - "nxt_size_add(SIZE_MAX, 1) did not overflow"); - NXT_TEST_CHECK(thr->log, nxt_size_mul(SIZE_MAX, 2, &r) != 0, - "nxt_size_mul(SIZE_MAX, 2) did not overflow"); + if (nxt_checked_run_table(thr, "nxt_size_add", nxt_size_add, + nxt_checked_add_cases, + nxt_nitems(nxt_checked_add_cases)) != NXT_OK + || nxt_checked_run_table(thr, "nxt_size_add_portable", + nxt_size_add_portable, nxt_checked_add_cases, + nxt_nitems(nxt_checked_add_cases)) != NXT_OK + || nxt_checked_run_table(thr, "nxt_size_mul", nxt_size_mul, + nxt_checked_mul_cases, + nxt_nitems(nxt_checked_mul_cases)) != NXT_OK + || nxt_checked_run_table(thr, "nxt_size_mul_portable", + nxt_size_mul_portable, nxt_checked_mul_cases, + nxt_nitems(nxt_checked_mul_cases)) != NXT_OK) + { + return NXT_ERROR; + } /* Two whole 4-byte records, then a partial tail of 0..3 bytes. */ @@ -52,6 +128,26 @@ nxt_checked_test(nxt_thread_t *thr) && dst[0] == 0xAA, "nxt_span_copy() accepted or wrote a partial tail"); + /* A copy of 0 bytes succeeds and changes neither the span nor dst. */ + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 0) == 0 + && span.pos == buf + 4 && span.end == buf + 6 + && dst[0] == 0xAA, + "nxt_span_copy() of 0 bytes failed on a non-empty span"); + + /* The same on an empty span with pos == end == NULL. */ + + nxt_span_init(&span, NULL, NULL); + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 0) == 0 + && span.pos == NULL && span.end == NULL + && dst[0] == 0xAA, + "nxt_span_copy() of 0 bytes failed on an empty span"); + + NXT_TEST_CHECK(thr->log, nxt_span_copy(&span, dst, 1) != 0 + && span.pos == NULL && dst[0] == 0xAA, + "nxt_span_copy() accepted a byte from an empty span"); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nxt_checked test passed"); return NXT_OK; } diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 8e29eeecc..c622a58e8 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -45,11 +45,21 @@ nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), _exit(fn(data)); } - if (child == -1 || waitpid(child, &status, 0) != child) { + if (child == -1) { nxt_log_alert(thr->log, "%s: fork() failed %E", name, nxt_errno); return -1; } + /* A signal may interrupt waitpid(). Then it is called again. */ + + while (waitpid(child, &status, 0) != child) { + if (nxt_errno != NXT_EINTR) { + nxt_log_alert(thr->log, "%s: waitpid() failed %E", name, + nxt_errno); + return -1; + } + } + if (!WIFEXITED(status)) { nxt_log_alert(thr->log, "%s: child killed by signal %d", name, WTERMSIG(status)); From 7db3eb34c1287c90d7f87811ca9d762795bdfc30 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 21:58:16 +0000 Subject: [PATCH 295/391] fix(port): bound the reads of untrusted shared-memory records An mmap message carries an array of nxt_port_mmap_msg_t records that the peer wrote. On the router side the peer is an application process. On the libunit side it is the router. Both walked the array with a raw pointer up to the end of the buffer. A buffer whose length was not a whole number of records made the last "record" read past the end. Walk the array with nxt_span_copy() on both sides, in nxt_port_mmap_read() and nxt_unit_mmap_read(). A partial tail is refused with an alert. Each record is copied out before a field of it is used. In libunit, fix the other checks around the incoming segments (#172): - nxt_unit_mmap_at() grows lib->incoming to fit a segment id from the peer. "i + 1 > cap" wrapped for i == UINT32_MAX, and the 1.5x growth step could wrap below the target and spin. Refuse an id whose slot cannot be represented. Size the realloc() with nxt_size_mul(). - nxt_unit_incoming_mmap() mapped a segment of the size that the descriptor had, but every chunk offset assumes PORT_MMAP_SIZE. Require exactly that size, as the router does. - Read the segment id from the header once. A duplicate id keeps the segment that is already in place, because buffers may point into it. - A field string of a request must lie past fields[], because nxt_unit_request_group_dup_fields() moves a field one slot on by subtracting from its offset. Compare with the pointer that nxt_unit_sptr_in_buf() checked, not with a second read of the sptr: the peer keeps writing the shared buffer. - When nxt_unit_get_mmap() cannot be sent, take the read buffer off the wait queue before the caller releases it. Hand the other buffers that wait on the same segment back to their contexts as pending, so the next read asks again and no context hangs. On the router side, nxt_port_mmap_get() and nxt_port_mmap_increase_buf() no longer test the busy sentinel chunk. The peer maps it writable and can clear it. nxt_unit_sptr_in_buf() moves to nxt_unit_sptr.h and returns the resolved pointer. It reads the offset once through volatile. The router can use the same check on an application's response. Tests: src/test/nxt_port_mmap_read_test.c drives nxt_port_mmap_read() with partial tails and out-of-range records in a child process, and releases what it builds so LeakSanitizer is quiet. src/test/nxt_unit_msg_test.c feeds crafted port messages through nxt_unit_test_process_msg(), a hook compiled in with --tests and for the fuzz build. fuzzing/nxt_unit_msg_fuzz.c is a libFuzzer target for libunit's receive side, with a seed corpus. It runs in CI with the other fuzzers. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check-test-hooks.sh | 2 +- .github/workflows/fuzzing.yml | 13 +- auto/fuzzing | 13 +- auto/make | 13 +- auto/sources | 8 + fuzzing/README.md | 17 +- fuzzing/build-fuzz.sh | 1 + .../mmap_partial_tail.bin | Bin 0 -> 51 bytes .../fuzz_unit_msg_seed_corpus/port_ack.bin | Bin 0 -> 18 bytes .../fuzz_unit_msg_seed_corpus/req_body.bin | Bin 0 -> 94 bytes .../req_headers_dup_field_inside_fields.bin | Bin 0 -> 198 bytes .../req_headers_fields_body.bin | Bin 0 -> 316 bytes .../req_headers_min.bin | Bin 0 -> 212 bytes .../req_headers_two_records.bin | Bin 0 -> 264 bytes .../rpc_ready_mmap.bin | Bin 0 -> 46 bytes fuzzing/fuzz_unit_msg_seed_corpus/shm_ack.bin | Bin 0 -> 18 bytes .../fuzz_unit_msg_seed_corpus/websocket.bin | Bin 0 -> 34 bytes fuzzing/nxt_unit_msg_fuzz.c | 439 ++++++++++++++ fuzzing/run-ci.sh | 5 +- src/nxt_port_memory.c | 52 +- src/nxt_unit.c | 367 +++++++++--- src/nxt_unit.h | 5 + src/nxt_unit_sptr.h | 49 ++ src/test/nxt_port_mmap_read_test.c | 327 ++++++++++ src/test/nxt_tests.c | 2 +- src/test/nxt_tests.h | 1 + src/test/nxt_unit_msg_test.c | 567 ++++++++++++++++++ 27 files changed, 1760 insertions(+), 121 deletions(-) create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/mmap_partial_tail.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/port_ack.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/req_body.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/req_headers_dup_field_inside_fields.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/req_headers_fields_body.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/req_headers_min.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/req_headers_two_records.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/rpc_ready_mmap.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/shm_ack.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/websocket.bin create mode 100644 fuzzing/nxt_unit_msg_fuzz.c create mode 100644 src/test/nxt_port_mmap_read_test.c create mode 100644 src/test/nxt_unit_msg_test.c diff --git a/.github/scripts/check-test-hooks.sh b/.github/scripts/check-test-hooks.sh index e5a56f6e2..32178c321 100755 --- a/.github/scripts/check-test-hooks.sh +++ b/.github/scripts/check-test-hooks.sh @@ -227,7 +227,7 @@ done # 3. The test programs took every linked object from the instrumented archive. programs="$build/tests $build/ncq_test $build/vbcq_test $build/unit_app_test \ - $build/unit_close_test $build/unit_port_recv_test \ + $build/unit_close_test $build/unit_port_recv_test $build/unit_msg_test \ $build/unit_websocket_chat $build/unit_websocket_echo" for prog in $programs; do diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index a0863aad0..cc5ec5b0b 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -14,6 +14,9 @@ on: # nxt_http_controller_fuzz.c #includes nxt_controller.c, so a change # there is a change to what this job fuzzes. - 'src/nxt_controller*' + - 'src/nxt_unit*' + - 'src/nxt_port_memory_int.h' + - 'src/nxt_span.h' # configure --fuzz, auto/fuzzing and auto/sources are what produce the # binaries; break them and the job has nothing to run. - configure @@ -27,7 +30,7 @@ on: required: false default: '60' targets: - description: 'Space-separated fuzzer names (empty = all five)' + description: 'Space-separated fuzzer names (empty = all six)' required: false default: '' @@ -46,10 +49,10 @@ jobs: libfuzzer: name: libFuzzer (ASan + UBSan) runs-on: ubuntu-latest - # Five targets at a 60s budget is a little over five minutes -- each + # Six targets at a 60s budget is a little over six minutes -- each # finishes within a second or two of its budget, and run-ci.sh caps the # worst case at two minutes each. The clang/ASan build is a few minutes - # on top. A dispatch can ask for a much longer budget across all five + # on top. A dispatch can ask for a much longer budget across all six # targets, so it gets a much longer bound; a pull request must not, or a # hung job holds a runner for an hour. timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 120 || 25 }} @@ -83,11 +86,11 @@ jobs: ./configure --no-regex --no-pcre2 --fuzz=-fsanitize=fuzzer make fuzz -j"$(nproc)" - # All five targets, on a pull request and on a dispatch alike. Picking a + # All six targets, on a pull request and on a dispatch alike. Picking a # subset per changed path was tried and dropped: every path filter above # already maps to a target, `fuzzing/**` maps to all of them, and a # target left out of the list is a harness that CI compiles but never - # runs. At about 61 seconds each the whole set is close to five minutes. + # runs. At about 61 seconds each the whole set is close to six minutes. - name: Run the fuzzers env: FUZZ_SECONDS: ${{ github.event.inputs.seconds || '60' }} diff --git a/auto/fuzzing b/auto/fuzzing index f792941bd..ffc2b643b 100644 --- a/auto/fuzzing +++ b/auto/fuzzing @@ -23,7 +23,8 @@ fuzz: $NXT_BUILD_DIR/fuzz_basic \\ $NXT_BUILD_DIR/fuzz_http_controller \\ $NXT_BUILD_DIR/fuzz_http_h1p \\ $NXT_BUILD_DIR/fuzz_http_h1p_peer \\ - $NXT_BUILD_DIR/fuzz_json + $NXT_BUILD_DIR/fuzz_json \\ + $NXT_BUILD_DIR/fuzz_unit_msg $NXT_BUILD_DIR/fuzz_basic: \$(NXT_FUZZ_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC @@ -70,6 +71,16 @@ $NXT_BUILD_DIR/fuzz_json: \$(NXT_FUZZ_OBJS) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ $NXT_FUZZ +# libunit's receive side: linked against libunit, not the core library. +$NXT_BUILD_DIR/fuzz_unit_msg: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_unit_msg \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_unit_msg_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + END fi diff --git a/auto/make b/auto/make index 1cc26788f..7d8ae3527 100644 --- a/auto/make +++ b/auto/make @@ -178,6 +178,7 @@ for nxt_src in $NXT_LIB_SRCS $NXT_TEST_SRCS $NXT_FUZZ_SRCS $NXT_LIB_UNIT_SRCS \ src/test/nxt_unit_app_test.c \ src/test/nxt_unit_close_test.c \ src/test/nxt_unit_port_recv_test.c \ + src/test/nxt_unit_msg_test.c \ src/test/nxt_unit_websocket_chat.c \ src/test/nxt_unit_websocket_echo.c do @@ -342,7 +343,8 @@ tests: $NXT_BUILD_DIR/tests \\ $NXT_BUILD_DIR/unit_app_test $NXT_BUILD_DIR/unit_websocket_chat \\ $NXT_BUILD_DIR/unit_websocket_echo \\ $NXT_BUILD_DIR/unit_close_test \\ - $NXT_BUILD_DIR/unit_port_recv_test + $NXT_BUILD_DIR/unit_port_recv_test \\ + $NXT_BUILD_DIR/unit_msg_test $NXT_BUILD_DIR/tests: \$(NXT_TEST_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_TEST_STATIC \\ @@ -396,6 +398,15 @@ $NXT_BUILD_DIR/unit_port_recv_test: \\ $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS +$NXT_BUILD_DIR/unit_msg_test: \\ + $NXT_BUILD_DIR/src/test/nxt_unit_msg_test.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/unit_msg_test \\ + \$(CFLAGS) $NXT_BUILD_DIR/src/test/nxt_unit_msg_test.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS + $NXT_BUILD_DIR/unit_websocket_chat: \\ $NXT_BUILD_DIR/src/test/nxt_unit_websocket_chat.o \\ $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_TEST_STATIC diff --git a/auto/sources b/auto/sources index 8a66fb270..d79948026 100644 --- a/auto/sources +++ b/auto/sources @@ -185,6 +185,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_fd_event_change_test.c \ src/test/nxt_port_use_unless_zero_test.c \ src/test/nxt_port_mmap_range_test.c \ + src/test/nxt_port_mmap_read_test.c \ src/test/nxt_port_ready_test.c \ src/test/nxt_router_new_port_test.c \ src/test/nxt_router_start_fail_test.c \ @@ -343,12 +344,19 @@ if [ "$NXT_TEST_BUILD" = "YES" ]; then fi +# libunit's message-feeding hook is compiled in for the fuzz targets too. +if [ -n "$NXT_FUZZ" ]; then + nxt_have=NXT_FUZZ_BUILD . auto/have +fi + + NXT_FUZZ_SRCS=" \ fuzzing/nxt_basic_fuzz.c \ fuzzing/nxt_http_controller_fuzz.c \ fuzzing/nxt_http_h1p_fuzz.c \ fuzzing/nxt_http_h1p_peer_fuzz.c \ fuzzing/nxt_json_fuzz.c \ + fuzzing/nxt_unit_msg_fuzz.c \ " diff --git a/fuzzing/README.md b/fuzzing/README.md index c0be7aecc..090860c33 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -42,12 +42,14 @@ $ mkdir -p build/fuzz_http_controller_seed $ mkdir -p build/fuzz_http_h1p_seed $ mkdir -p build/fuzz_http_h1p_peer_seed $ mkdir -p build/fuzz_json_seed +$ mkdir -p build/fuzz_unit_msg_seed $ ./build/fuzz_basic build/fuzz_basic_seed fuzzing/fuzz_basic_seed_corpus $ ./build/fuzz_http_controller build/fuzz_http_controller_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_http_h1p build/fuzz_http_h1p_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_http_h1p_peer build/fuzz_http_h1p_peer_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_json build/fuzz_json_seed fuzzing/fuzz_json_seed_corpus +$ ./build/fuzz_unit_msg build/fuzz_unit_msg_seed fuzzing/fuzz_unit_msg_seed_corpus ``` Here is more information about [LibFuzzer](https://llvm.org/docs/LibFuzzer.html). @@ -67,6 +69,11 @@ may vary). clang, llvm & compiler-rt ``` +`fuzz_unit_msg` (`nxt_unit_msg_fuzz.c`) is linked against libunit rather than +the core library: it feeds one router-to-application port message through +`nxt_unit_process_msg()` and the request handler. Its input format and what +the harness steers are described at the top of the file. + ## In CI `.github/workflows/fuzzing.yml` builds these targets with clang, ASan and @@ -74,15 +81,15 @@ UBSan and runs `fuzzing/run-ci.sh`, which gives each target a bounded libFuzzer budget against its seed corpus and fails on the first crash. ```shell -$ fuzzing/run-ci.sh # all five, 60s each +$ fuzzing/run-ci.sh # all six, 60s each $ fuzzing/run-ci.sh -t 120 fuzz_http_h1p # one target, longer ``` A pull request that touches `src/nxt_http*`, `src/nxt_h1proto*`, -`src/nxt_controller*`, `src/nxt_conf*` or `fuzzing/` runs all five targets -- -roughly five minutes; a manual `workflow_dispatch` takes a budget and a -target list as inputs. Reproducers land in `build/fuzz-artifacts/` and are -uploaded as an artifact when the job fails. +`src/nxt_controller*`, `src/nxt_conf*`, `src/nxt_unit*` or `fuzzing/` runs +all six targets -- roughly six minutes; a manual `workflow_dispatch` takes a +budget and a target list as inputs. Reproducers land in +`build/fuzz-artifacts/` and are uploaded as an artifact when the job fails. Each target finishes within a second or two of its budget. `run-ci.sh` caps it at the budget plus a minute anyway -- libFuzzer only checks the budget diff --git a/fuzzing/build-fuzz.sh b/fuzzing/build-fuzz.sh index 62f7a6761..a1bee2b3a 100644 --- a/fuzzing/build-fuzz.sh +++ b/fuzzing/build-fuzz.sh @@ -14,6 +14,7 @@ mkdir -p build/fuzz_http_controller_seed mkdir -p build/fuzz_http_h1p_seed mkdir -p build/fuzz_http_h1p_peer_seed mkdir -p build/fuzz_json_seed +mkdir -p build/fuzz_unit_msg_seed echo "" echo "Run: ./build/\${fuzzer} build/\${fuzzer}_seed fuzzing/\${fuzzer}_seed_corpus" diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/mmap_partial_tail.bin b/fuzzing/fuzz_unit_msg_seed_corpus/mmap_partial_tail.bin new file mode 100644 index 0000000000000000000000000000000000000000..7372259db81b28d3c3f0153295b8c3b0d336b0d2 GIT binary patch literal 51 ecmY#n;Adc9V1!^s5DfzYPzGZoQ$}V55&!^0ss}m% literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/port_ack.bin b/fuzzing/fuzz_unit_msg_seed_corpus/port_ack.bin new file mode 100644 index 0000000000000000000000000000000000000000..8e0b7ecb4c5026a15d5a87c0ee125906d6878ca4 GIT binary patch literal 18 QcmWe&00BlY<_A#>00E@{BLDyZ literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_body.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_body.bin new file mode 100644 index 0000000000000000000000000000000000000000..05da4caba4c11eedc327850105be66a02ea7e856 GIT binary patch literal 94 acmb1P;9+23U<6}HMi31H4nRf)g#ZBfavp{N literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_dup_field_inside_fields.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_dup_field_inside_fields.bin new file mode 100644 index 0000000000000000000000000000000000000000..75c9e47f3fb9c962f4c25072d33c1904fb33f30a GIT binary patch literal 198 zcmb1P;9+23U<6}vMi31HD}W4U4j2!WW(IQqqXA?U5kL+I#{zLA5QhSBAQ1Zku_q9_ z06b0bp4`LRvNMj?u!J0(;Ys5lCi!Chj0*O0p#5faiP@@(;nXk5s$XEqHn55t8Qv;zd4LEGo|e;WEjAE{YJ zX<~L3UNnwJOUz|{w-ckS()KgVGX<-T*2~gpr7Pj)5&n;xgZWK>#pbYtHEduHJJ`Y<`Zk&T z_5vap*wk#GLs-FVDk+dMay`c*uQzlAzW?P5Ur?IIDm~>FO|BXrMll7vf7eN#=_J!# Qc`Fjq$yK{Ws?Hg`0E6=_`~Uy| literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/rpc_ready_mmap.bin b/fuzzing/fuzz_unit_msg_seed_corpus/rpc_ready_mmap.bin new file mode 100644 index 0000000000000000000000000000000000000000..5812bfb2b529d00fbeae1c427616df80678eea26 GIT binary patch literal 46 dcmb1P;A3E5V1!^s5DfzYKt>}|MrH*P002QQ2QvTw literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/shm_ack.bin b/fuzzing/fuzz_unit_msg_seed_corpus/shm_ack.bin new file mode 100644 index 0000000000000000000000000000000000000000..e45448a7646fcb08ef75ecbda950e7702d588159 GIT binary patch literal 18 QcmWe&00BlYmIqM`00IR7GXMYp literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/websocket.bin b/fuzzing/fuzz_unit_msg_seed_corpus/websocket.bin new file mode 100644 index 0000000000000000000000000000000000000000..fb7368938101d70fd3fa4e2a04d50625cfb90012 GIT binary patch literal 34 bcmb1P;9+23U<6|+Mi31HEI>vhQ${8L3swQ2 literal 0 HcmV?d00001 diff --git a/fuzzing/nxt_unit_msg_fuzz.c b/fuzzing/nxt_unit_msg_fuzz.c new file mode 100644 index 000000000..9c8ed25b5 --- /dev/null +++ b/fuzzing/nxt_unit_msg_fuzz.c @@ -0,0 +1,439 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * libFuzzer target for libunit's receive side: one port message, as the + * router would send it, through nxt_unit_process_msg() -- the mmap record + * walk (nxt_unit_mmap_read()), request arrival validation + * (nxt_unit_process_req_headers(), nxt_unit_sptr_in_buf()), request body + * and websocket frames -- and, for a request that comes out ready, the + * application's request handler, which reads every field and finishes it. + * + * Input layout: + * + * [0..1] little-endian length L of the port message (capped at the + * 16 KB read buffer) + * [2..2+L) the port message: nxt_port_msg_t, then its payload -- for + * an mmap message an array of nxt_port_mmap_msg_t + * [2+L..) the shared memory payload (up to 1 MB), placed in chunks + * of segment 0 that the records then point into + * + * The trust model is libunit's: the router is the peer. The harness plays + * a correct router for everything that is its job rather than the + * message's, and leaves the message itself to the fuzzer: + * + * - Segment 0 is a real shared memory segment, handed to libunit the way + * the router hands one over. The payload goes into chunks the harness + * allocates in the segment's free map, as the router does, and a record + * whose (chunk_id, size) is in range is moved into that allocation: + * without that, the next input overwrites chunks a request that is + * still waiting for its body holds -- memory the router would never + * reuse -- and the "finding" is the harness's own. Out-of-range + * records are left alone, to exercise the range check. + * - Every whole record's mmap_id is forced to 0 unless it is 0xFFFFFFFF. + * Any other id names a segment libunit does not have, and libunit then + * (correctly) parks the message and asks the router for it; parked + * messages would pile up across runs. The id is not capped by design + * (#172), so an id near 2^32 would also grow the segment array to that + * size -- an allocation, not a finding. + * - QUIT and REMOVE_PID are skipped: they tear down state every later run + * depends on. + * - The sender pid and reply port are set to the router port libunit was + * started with, so that a request can complete instead of waiting for a + * port that never arrives. + * + * Requests that wait for a body keep their chunks, and nothing may ever + * send that body; when the segment runs out, the whole libunit instance is + * torn down (which finishes those requests) and set up again. + * + * Built only with --fuzz, which also compiles in the + * nxt_unit_test_process_msg() hook (NXT_FUZZ_BUILD). + */ + +#include +#include +#include +#include +#include + +#include +#include + + +#define NXT_UNIT_MSG_FUZZ_MSG_MAX 16384 +#define NXT_UNIT_MSG_FUZZ_DATA_MAX (1024 * 1024) +#define NXT_UNIT_MSG_FUZZ_ROUTER_PORT 2 + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + + +static nxt_unit_ctx_t *nxt_unit_msg_fuzz_ctx; +static nxt_port_mmap_header_t *nxt_unit_msg_fuzz_hdr; +static int nxt_unit_msg_fuzz_fds[3] = { -1, -1, -1 }; +static int nxt_unit_msg_fuzz_null = -1; +static volatile uint64_t nxt_unit_msg_fuzz_sink; + + +static u_char +nxt_unit_msg_fuzz_last(nxt_unit_sptr_t *sptr, uint32_t length) +{ + return length == 0 ? 0 + : ((u_char *) nxt_unit_sptr_get(sptr))[length - 1]; +} + + +static void +nxt_unit_msg_fuzz_handler(nxt_unit_request_info_t *req) +{ + uint32_t i; + uint64_t sum; + nxt_unit_field_t *f; + nxt_unit_request_t *r; + + /* As the Python and Java modules do before reading the fields. */ + nxt_unit_request_group_dup_fields(req); + + r = req->request; + sum = 0; + + /* + * Touch what an application reads: the last byte of every string the + * arrival checks vetted, at the length they vetted. The request lives + * in the shared segment, which ASan does not shadow, so what this + * catches is a read that leaves the mapping altogether. + */ + sum += nxt_unit_msg_fuzz_last(&r->method, r->method_length); + sum += nxt_unit_msg_fuzz_last(&r->version, r->version_length); + sum += nxt_unit_msg_fuzz_last(&r->remote, r->remote_length); + sum += nxt_unit_msg_fuzz_last(&r->local_addr, r->local_addr_length); + sum += nxt_unit_msg_fuzz_last(&r->local_port, r->local_port_length); + sum += nxt_unit_msg_fuzz_last(&r->server_name, r->server_name_length); + sum += nxt_unit_msg_fuzz_last(&r->target, r->target_length); + sum += nxt_unit_msg_fuzz_last(&r->path, r->path_length); + sum += nxt_unit_msg_fuzz_last(&r->query, r->query_length); + + for (i = 0; i < r->fields_count; i++) { + f = &r->fields[i]; + + sum += nxt_unit_msg_fuzz_last(&f->name, f->name_length); + sum += nxt_unit_msg_fuzz_last(&f->value, f->value_length); + } + + if (r->content_length_field != NXT_UNIT_NONE_FIELD) { + sum += r->fields[r->content_length_field].hash; + } + + if (r->content_type_field != NXT_UNIT_NONE_FIELD) { + sum += r->fields[r->content_type_field].hash; + } + + if (r->cookie_field != NXT_UNIT_NONE_FIELD) { + sum += r->fields[r->cookie_field].hash; + } + + if (r->authorization_field != NXT_UNIT_NONE_FIELD) { + sum += r->fields[r->authorization_field].hash; + } + + nxt_unit_msg_fuzz_sink = sum; + + nxt_unit_request_done(req, NXT_UNIT_ERROR); +} + + +static ssize_t +nxt_unit_msg_fuzz_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + const void *buf, size_t buf_size, const void *oob, size_t oob_size) +{ + return buf_size; +} + + +static ssize_t +nxt_unit_msg_fuzz_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *buf, + size_t buf_size, void *oob, size_t *oob_size) +{ + *oob_size = 0; + + return 0; +} + + +static int +nxt_unit_msg_fuzz_shm(size_t size) +{ + int fd; + + fd = syscall(SYS_memfd_create, "unit.msg.fuzz", 0); + if (fd == -1) { + return -1; + } + + if (ftruncate(fd, size) == -1) { + close(fd); + return -1; + } + + return fd; +} + + +/* A libunit instance with segment 0 handed over, as a router would. */ +static int +nxt_unit_msg_fuzz_setup(void) +{ + int fd, ready[2], router[2], read[2], shared[2]; + nxt_port_msg_t msg; + nxt_unit_init_t init; + nxt_port_mmap_header_t *hdr; + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, ready) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, router) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, read) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, shared) == -1) + { + return -1; + } + + /* The peer ends, which libunit is not given and does not close. */ + nxt_unit_msg_fuzz_fds[0] = ready[1]; + nxt_unit_msg_fuzz_fds[1] = router[1]; + nxt_unit_msg_fuzz_fds[2] = shared[1]; + + memset(&init, 0, sizeof(init)); + + init.callbacks.request_handler = nxt_unit_msg_fuzz_handler; + init.callbacks.port_send = nxt_unit_msg_fuzz_send; + init.callbacks.port_recv = nxt_unit_msg_fuzz_recv; + + init.ready_port.id.pid = getpid(); + init.ready_port.id.id = 1; + init.ready_port.in_fd = -1; + init.ready_port.out_fd = ready[0]; + init.ready_stream = 1; + + init.router_port.id.pid = getpid(); + init.router_port.id.id = NXT_UNIT_MSG_FUZZ_ROUTER_PORT; + init.router_port.in_fd = -1; + init.router_port.out_fd = router[0]; + + init.read_port.id.pid = getpid(); + init.read_port.id.id = 3; + init.read_port.in_fd = read[0]; + init.read_port.out_fd = read[1]; + + init.shared_port_fd = shared[0]; + init.shared_queue_fd = nxt_unit_msg_fuzz_shm(sizeof(nxt_app_queue_t)); + + /* libunit's own log lines would drown libFuzzer's. */ + init.log_fd = nxt_unit_msg_fuzz_null; + + if (init.shared_queue_fd == -1) { + return -1; + } + + nxt_unit_msg_fuzz_ctx = nxt_unit_init(&init); + if (nxt_unit_msg_fuzz_ctx == NULL) { + return -1; + } + + fd = nxt_unit_msg_fuzz_shm(PORT_MMAP_SIZE); + if (fd == -1) { + return -1; + } + + hdr = mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + if (hdr == MAP_FAILED) { + return -1; + } + + /* As nxt_port_new_port_mmap() initialises a segment. */ + memset(hdr->free_map, 0xFF, sizeof(hdr->free_map)); + nxt_port_mmap_set_chunk_busy(hdr->free_map, PORT_MMAP_CHUNK_COUNT); + + hdr->id = 0; + hdr->src_pid = getpid(); + hdr->dst_pid = getpid(); + hdr->sent_over = 0xFFFFu; + + nxt_unit_msg_fuzz_hdr = hdr; + + memset(&msg, 0, sizeof(msg)); + + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_MMAP; + msg.last = 1; + + /* libunit maps the segment itself and closes the descriptor. */ + if (nxt_unit_test_process_msg(nxt_unit_msg_fuzz_ctx, &msg, sizeof(msg), + fd) + != NXT_UNIT_OK) + { + return -1; + } + + return 0; +} + + +static int +nxt_unit_msg_fuzz_reset(void) +{ + int i; + + nxt_unit_done(nxt_unit_msg_fuzz_ctx); + + munmap(nxt_unit_msg_fuzz_hdr, PORT_MMAP_SIZE); + + for (i = 0; i < 3; i++) { + close(nxt_unit_msg_fuzz_fds[i]); + nxt_unit_msg_fuzz_fds[i] = -1; + } + + return nxt_unit_msg_fuzz_setup(); +} + + +/* "n" consecutive free chunks, marked busy, as the router allocates. */ +static int +nxt_unit_msg_fuzz_alloc(size_t n, nxt_chunk_id_t *base) +{ + nxt_free_map_t *m; + nxt_chunk_id_t c, i; + + m = nxt_unit_msg_fuzz_hdr->free_map; + + for (c = 0; c + n <= PORT_MMAP_CHUNK_COUNT; c += i + 1) { + + for (i = 0; i < n; i++) { + if ((m[FREE_IDX(c + i)] & FREE_MASK(c + i)) == 0) { + break; + } + } + + if (i == n) { + for (i = 0; i < n; i++) { + nxt_port_mmap_set_chunk_busy(m, c + i); + } + + *base = c; + + return 0; + } + } + + return -1; +} + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + nxt_unit_msg_fuzz_null = open("/dev/null", O_WRONLY); + + if (nxt_unit_msg_fuzz_null == -1 || nxt_unit_msg_fuzz_setup() != 0) { + abort(); + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + u_char buf[NXT_UNIT_MSG_FUZZ_MSG_MAX]; + size_t len, dlen, n, off, nchunks, room; + nxt_chunk_id_t base, c; + nxt_port_msg_t *msg; + nxt_port_mmap_msg_t rec; + + if (size < 2) { + return 0; + } + + len = data[0] | (data[1] << 8); + data += 2; + size -= 2; + + len = nxt_min(len, nxt_min(size, sizeof(buf))); + + memcpy(buf, data, len); + + if (len < sizeof(nxt_port_msg_t)) { + /* Still worth sending: the "too small" path. */ + (void) nxt_unit_test_process_msg(nxt_unit_msg_fuzz_ctx, buf, len, -1); + return 0; + } + + msg = (nxt_port_msg_t *) buf; + + if (msg->type == _NXT_PORT_MSG_QUIT + || msg->type == _NXT_PORT_MSG_REMOVE_PID) + { + return 0; + } + + /* + * The reply goes to the router port libunit was started with; any other + * (pid, port) is unknown, and a request for it waits for a NEW_PORT + * that never comes. + */ + msg->pid = getpid(); + msg->reply_port = NXT_UNIT_MSG_FUZZ_ROUTER_PORT; + + base = 0; + n = 0; + + if (msg->mmap && len >= sizeof(nxt_port_msg_t) + + sizeof(nxt_port_mmap_msg_t)) + { + dlen = nxt_min(size - len, NXT_UNIT_MSG_FUZZ_DATA_MAX); + n = (dlen + PORT_MMAP_CHUNK_SIZE - 1) / PORT_MMAP_CHUNK_SIZE; + n = nxt_max(n, 1); + + if (nxt_unit_msg_fuzz_alloc(n, &base) != 0) { + if (nxt_unit_msg_fuzz_reset() != 0) { + abort(); + } + + if (nxt_unit_msg_fuzz_alloc(n, &base) != 0) { + abort(); + } + } + + memcpy(nxt_port_mmap_chunk_start(nxt_unit_msg_fuzz_hdr, base), + data + len, dlen); + + for (off = sizeof(nxt_port_msg_t); + off + sizeof(nxt_port_mmap_msg_t) <= len; + off += sizeof(nxt_port_mmap_msg_t)) + { + memcpy(&rec, buf + off, sizeof(rec)); + + if (rec.mmap_id != 0xFFFFFFFF) { + rec.mmap_id = 0; + } + + if (nxt_port_mmap_chunk_range_valid(rec.chunk_id, rec.size, + &nchunks)) + { + c = base + rec.chunk_id % n; + room = (base + n - c) * PORT_MMAP_CHUNK_SIZE; + + rec.chunk_id = c; + rec.size = nxt_min(rec.size, room); + } + + memcpy(buf + off, &rec, sizeof(rec)); + } + } + + (void) nxt_unit_test_process_msg(nxt_unit_msg_fuzz_ctx, buf, len, -1); + + return 0; +} diff --git a/fuzzing/run-ci.sh b/fuzzing/run-ci.sh index 50c37bb58..eeabfd7fb 100755 --- a/fuzzing/run-ci.sh +++ b/fuzzing/run-ci.sh @@ -10,7 +10,7 @@ # fuzzing/run-ci.sh [-t SECONDS] [TARGET ...] # # -t SECONDS libFuzzer budget per target (default 60) -# TARGET ... fuzzer names; default is all five +# TARGET ... fuzzer names; default is all six # # Expects the fuzzers to be built already: # CC=clang CXX=clang++ \ @@ -47,6 +47,7 @@ corpus_for() { case "$1" in fuzz_basic) echo fuzz_basic_seed_corpus ;; fuzz_json) echo fuzz_json_seed_corpus ;; + fuzz_unit_msg) echo fuzz_unit_msg_seed_corpus ;; fuzz_http_*) echo fuzz_http_seed_corpus ;; *) die "unknown target: $1" ;; esac @@ -60,7 +61,7 @@ dict_for() { } [ $# -gt 0 ] || set -- fuzz_basic fuzz_json fuzz_http_controller \ - fuzz_http_h1p fuzz_http_h1p_peer + fuzz_http_h1p fuzz_http_h1p_peer fuzz_unit_msg # Freeze the list: the loop below rebuilds "$@" to hold each target's libFuzzer # arguments, so it cannot also be the list being iterated. diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index cead5e39b..2cde6acb2 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -5,6 +5,7 @@ */ #include +#include #if (NXT_HAVE_MEMFD_CREATE) @@ -581,7 +582,10 @@ nxt_port_mmap_get(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_chunk_id_t *c, nchunks = 1; while (nchunks < n) { - res = nxt_port_mmap_chk_set_chunk_busy(free_map, *c + nchunks); + /* Not up to the sentinel: see nxt_port_mmap_increase_buf(). */ + res = *c + nchunks < PORT_MMAP_CHUNK_COUNT + && nxt_port_mmap_chk_set_chunk_busy(free_map, + *c + nchunks); if (res == 0) { for (i = 0; i < nchunks; i++) { @@ -764,8 +768,11 @@ nxt_port_mmap_increase_buf(nxt_task_t *task, nxt_buf_t *b, size_t size, c = start; - /* Try to acquire as much chunks as required. */ - while (nchunks > 0) { + /* + * Try to acquire as much chunks as required. Not up to the busy + * sentinel: the peer maps the segment writable and can clear it. + */ + while (nchunks > 0 && c < PORT_MMAP_CHUNK_COUNT) { if (nxt_port_mmap_chk_set_chunk_busy(hdr->free_map, c) == 0) { break; @@ -911,27 +918,51 @@ nxt_port_mmap_write(nxt_task_t *task, nxt_port_t *port, } +/* + * The buffers of an mmap message carry an array of nxt_port_mmap_msg_t + * written by the peer, which may be an untrusted application process. The + * array is walked with nxt_span_copy(), so a buffer whose length is not a + * whole number of records -- a partial tail of 1 to 11 bytes -- is refused + * at the tail instead of being read as a record that runs past mem.free. + * Each record is copied out before any field is used; the fields are then + * bounds-checked by nxt_port_mmap_get_incoming_buf(): mmap_id against the + * sender's incoming segments, chunk_id and size by + * nxt_port_mmap_chunk_range_valid(). + */ + void nxt_port_mmap_read(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_buf_t *b, **pb; - nxt_port_mmap_msg_t *end, *mmap_msg; + nxt_span_t span; + nxt_port_mmap_msg_t mmap_msg; pb = &msg->buf; msg->size = 0; for (b = msg->buf; b != NULL; b = b->next) { - mmap_msg = (nxt_port_mmap_msg_t *) b->mem.pos; - end = (nxt_port_mmap_msg_t *) b->mem.free; + nxt_span_init(&span, b->mem.pos, b->mem.free); + + while (nxt_span_len(&span) != 0) { + + if (nxt_slow_path(nxt_span_copy(&span, &mmap_msg, + sizeof(nxt_port_mmap_msg_t)) + != 0)) + { + nxt_alert(task, "invalid mmap message from pid %PI: " + "%uz trailing bytes are not a whole record", + msg->port_msg.pid, nxt_span_len(&span)); + + break; + } - while (mmap_msg < end) { nxt_debug(task, "mmap_msg={%D, %D, %D} from %PI", - mmap_msg->mmap_id, mmap_msg->chunk_id, mmap_msg->size, + mmap_msg.mmap_id, mmap_msg.chunk_id, mmap_msg.size, msg->port_msg.pid); *pb = nxt_port_mmap_get_incoming_buf(task, msg->port, - msg->port_msg.pid, mmap_msg); + msg->port_msg.pid, &mmap_msg); if (nxt_slow_path(*pb == NULL)) { nxt_log_error(NXT_LOG_ERR, task->log, "failed to get mmap buffer"); @@ -939,9 +970,8 @@ nxt_port_mmap_read(nxt_task_t *task, nxt_port_recv_msg_t *msg) break; } - msg->size += mmap_msg->size; + msg->size += mmap_msg.size; pb = &(*pb)->next; - mmap_msg++; /* Mark original buf as complete. */ b->mem.pos += sizeof(nxt_port_mmap_msg_t); diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 077b5ab79..26f18ae07 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -5,6 +5,8 @@ #include "nxt_main.h" #include "nxt_port_memory_int.h" +#include "nxt_checked.h" +#include "nxt_span.h" #include "nxt_socket_msg.h" #include "nxt_port_queue.h" #include "nxt_app_queue.h" @@ -142,6 +144,8 @@ static int nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, static int nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_read_buf_t *rbuf); static int nxt_unit_get_mmap(nxt_unit_ctx_t *ctx, pid_t pid, uint32_t id); +static void nxt_unit_unpark_rbufs(nxt_unit_ctx_t *ctx, + nxt_queue_t *awaiting_rbuf); static void nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, nxt_port_mmap_header_t *hdr, void *start, uint32_t size); static int nxt_unit_send_shm_ack(nxt_unit_ctx_t *ctx, pid_t pid); @@ -266,53 +270,10 @@ nxt_unit_response_buf_size(uint32_t max_fields_count, /* - * Validate that an sptr field within a peer-supplied buffer dereferences - * to a [length]-byte range that is wholly inside the buffer. Used at - * request-arrival time to vet every sptr in nxt_unit_request_t before - * the application sees it. - * - * sptr->base aliases the address of the sptr itself (the union encodes - * an offset relative to that location), so this also implicitly checks - * that the sptr is inside the buffer. + * nxt_unit_sptr_in_buf() moved to nxt_unit_sptr.h: the router's response + * parser (src/nxt_router.c) needs the exact same bounds check against a + * peer-supplied buffer, on the other side of the trust boundary. */ -static int -nxt_unit_sptr_in_buf(nxt_unit_sptr_t *sptr, uint32_t length, - void *buf_start, uint32_t buf_size) -{ - size_t sptr_off, end_off; - - if ((uint8_t *) sptr < (uint8_t *) buf_start) { - return 0; - } - - sptr_off = (uint8_t *) sptr - (uint8_t *) buf_start; - - /* - * The sptr struct itself must fit inside the buffer before we - * dereference sptr->offset. Reject a buffer too small to hold an - * sptr first: buf_size is uint32_t and sizeof() is size_t, so - * "buf_size - sizeof(nxt_unit_sptr_t)" is evaluated in size_t and - * underflows to a huge value -- wrongly passing the bound check -- - * when buf_size is smaller than the struct. The short-circuit keeps - * the subtraction below from ever underflowing. - */ - if (buf_size < sizeof(nxt_unit_sptr_t) - || sptr_off > buf_size - sizeof(nxt_unit_sptr_t)) - { - return 0; - } - - if (sptr->offset > buf_size - sptr_off) { - return 0; - } - - end_off = sptr_off + sptr->offset; - if (length > buf_size - end_off) { - return 0; - } - - return 1; -} struct nxt_unit_mmap_buf_s { @@ -1334,6 +1295,73 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf, } +#if (NXT_TESTS || NXT_FUZZ_BUILD) + +/* + * Feeds one message through nxt_unit_process_msg() as if it had just been + * read from a port of "ctx", then reads the buffers it made pending and + * runs whatever request it made ready through callbacks.request_handler, + * the way the read loops do. "fd", when + * not -1, arrives as the message's SCM_RIGHTS descriptor and is owned by + * libunit from here on. Used by src/test/nxt_unit_msg_test.c and + * fuzzing/nxt_unit_msg_fuzz.c. + */ + +int +nxt_unit_test_process_msg(nxt_unit_ctx_t *ctx, const void *msg, size_t size, + int fd) +{ + int rc; + struct cmsghdr *cmsg; + nxt_unit_read_buf_t *rbuf; + + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + return NXT_UNIT_ERROR; + } + + if (nxt_slow_path(size > sizeof(rbuf->buf))) { + nxt_unit_read_buf_release(ctx, rbuf); + + return NXT_UNIT_ERROR; + } + + /* Deterministic bytes past the message, whatever the buffer held. */ + memset(rbuf->buf, 0, sizeof(rbuf->buf)); + memcpy(rbuf->buf, msg, size); + + rbuf->size = size; + rbuf->oob.size = 0; + rbuf->oob.truncated = 0; + + if (fd != -1) { + cmsg = (struct cmsghdr *) rbuf->oob.buf; + + memset(cmsg, 0, CMSG_SPACE(sizeof(int))); + + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + + memcpy(CMSG_DATA(cmsg), &fd, sizeof(int)); + + rbuf->oob.size = CMSG_SPACE(sizeof(int)); + } + + rc = nxt_unit_process_msg(ctx, rbuf, NULL); + + if (rc != NXT_UNIT_ERROR) { + rc = nxt_unit_process_pending_rbuf(ctx); + } + + nxt_unit_process_ready_req(ctx); + + return rc; +} + +#endif + + static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) { @@ -1471,6 +1499,7 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, * co-located with arrival makes the trust boundary explicit. */ { + void *name, *value, *end; uint32_t i; nxt_unit_request_t *vr = recv_msg->start; uint32_t vsize = recv_msg->size; @@ -1521,14 +1550,26 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, return NXT_UNIT_ERROR; } + /* + * Field strings must also lie past fields[]: the router puts them + * there, and nxt_unit_request_group_dup_fields() moves fields one + * slot on by subtracting sizeof(nxt_unit_field_t) from their + * offsets, which a target inside fields[] would underflow. The + * pointers the check returns are used, not the sptr read again: + * the peer can change the offset between two reads. + */ + end = &vr->fields[vr->fields_count]; + for (i = 0; i < vr->fields_count; i++) { - if (nxt_slow_path( - !nxt_unit_sptr_in_buf(&vr->fields[i].name, - vr->fields[i].name_length, - recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->fields[i].value, + name = nxt_unit_sptr_in_buf(&vr->fields[i].name, + vr->fields[i].name_length, + recv_msg->start, vsize); + value = nxt_unit_sptr_in_buf(&vr->fields[i].value, vr->fields[i].value_length, - recv_msg->start, vsize))) + recv_msg->start, vsize); + + if (nxt_slow_path(name == NULL || value == NULL + || name < end || value < end)) { nxt_unit_warn(ctx, "#%"PRIu32": malformed request: field " "%"PRIu32" sptr out of buffer", @@ -4425,6 +4466,7 @@ nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx) static nxt_unit_mmap_t * nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i) { + size_t bytes; uint32_t cap, n; nxt_unit_mmap_t *e; @@ -4432,25 +4474,50 @@ nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i) return mmaps->elts + i; } + /* + * The same guards as nxt_port_mmap_at() on the router side. "i" is a + * segment id taken from the peer (an mmap record or a segment header), + * and is deliberately not capped: the router allocates its outgoing + * segments without a limit. What has to hold is the arithmetic: a + * capacity able to hold slot i is i + 1 elements, which does not fit + * uint32_t for i == UINT32_MAX -- "i + 1 > cap" then wraps to 0, skips + * the growth, and the element pointer lands 4G elements past the array. + */ + if (nxt_slow_path(i == UINT32_MAX)) { + return NULL; + } + cap = mmaps->cap; if (cap == 0) { cap = i + 1; } - while (i + 1 > cap) { + while (cap <= i) { if (cap < 16) { cap = cap * 2; } else { + /* The 1.5x step would wrap below the target and spin. */ + if (nxt_slow_path(cap > UINT32_MAX - cap / 2)) { + return NULL; + } + cap = cap + cap / 2; } } if (cap != mmaps->cap) { - e = realloc(mmaps->elts, cap * sizeof(nxt_unit_mmap_t)); + if (nxt_slow_path(nxt_size_mul(cap, sizeof(nxt_unit_mmap_t), + &bytes) + != 0)) + { + return NULL; + } + + e = realloc(mmaps->elts, bytes); if (nxt_slow_path(e == NULL)) { return NULL; } @@ -4732,12 +4799,11 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) { int rc; void *mem; + uint32_t id; nxt_queue_t awaiting_rbuf; struct stat mmap_stat; nxt_unit_mmap_t *mm; nxt_unit_impl_t *lib; - nxt_unit_ctx_impl_t *ctx_impl; - nxt_unit_read_buf_t *rbuf; nxt_port_mmap_header_t *hdr; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -4751,7 +4817,20 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) return NXT_UNIT_ERROR; } - mem = mmap(NULL, mmap_stat.st_size, PROT_READ | PROT_WRITE, + /* + * Every chunk offset is computed against PORT_MMAP_SIZE, and the + * munmap() calls use it too: a shorter object faults on access, a + * longer one leaks the excess mapping. The router side requires the + * same (nxt_port_incoming_port_mmap()). + */ + if (nxt_slow_path(mmap_stat.st_size != (off_t) PORT_MMAP_SIZE)) { + nxt_unit_alert(ctx, "incoming_mmap: unexpected segment size: %d != %d", + (int) mmap_stat.st_size, (int) PORT_MMAP_SIZE); + + return NXT_UNIT_ERROR; + } + + mem = mmap(NULL, PORT_MMAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (nxt_slow_path(mem == MAP_FAILED)) { nxt_unit_alert(ctx, "incoming_mmap: mmap() failed: %s (%d)", @@ -4773,11 +4852,19 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) return NXT_UNIT_ERROR; } + /* + * The segment id lives in memory the sender keeps mapped writable: read + * it once and use only the copy. It indexes lib->incoming, which + * nxt_unit_mmap_at() grows to fit and which refuses an id whose slot + * cannot be represented. + */ + id = hdr->id; + nxt_queue_init(&awaiting_rbuf); pthread_mutex_lock(&lib->incoming.mutex); - mm = nxt_unit_mmap_at(&lib->incoming, hdr->id); + mm = nxt_unit_mmap_at(&lib->incoming, id); if (nxt_slow_path(mm == NULL)) { nxt_unit_alert(ctx, "incoming_mmap: failed to add to incoming array"); @@ -4785,6 +4872,18 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) rc = NXT_UNIT_ERROR; + } else if (nxt_slow_path(mm->hdr != NULL)) { + /* + * A duplicate id: buffers may point into the segment already + * there, so it stays and the new mapping goes. + */ + nxt_unit_warn(ctx, "incoming_mmap: duplicate segment id %"PRIu32, + id); + + munmap(mem, PORT_MMAP_SIZE); + + rc = NXT_UNIT_OK; + } else { mm->hdr = hdr; @@ -4798,7 +4897,24 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) pthread_mutex_unlock(&lib->incoming.mutex); - nxt_queue_each(rbuf, &awaiting_rbuf, nxt_unit_read_buf_t, link) { + nxt_unit_unpark_rbufs(ctx, &awaiting_rbuf); + + return rc; +} + + +/* + * Hands the read buffers parked on a segment back to their contexts as + * pending, so each context reads its message again. + */ + +static void +nxt_unit_unpark_rbufs(nxt_unit_ctx_t *ctx, nxt_queue_t *awaiting_rbuf) +{ + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_read_buf_t *rbuf; + + nxt_queue_each(rbuf, awaiting_rbuf, nxt_unit_read_buf_t, link) { ctx_impl = rbuf->ctx_impl; @@ -4813,8 +4929,6 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) nxt_unit_awake_ctx(ctx, ctx_impl); } nxt_queue_loop; - - return rc; } @@ -4903,6 +5017,7 @@ nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps, nxt_unit_read_buf_t *rbuf) { int res, need_rbuf; + nxt_queue_t awaiting_rbuf; nxt_unit_mmap_t *mm; nxt_unit_ctx_impl_t *ctx_impl; @@ -4936,7 +5051,40 @@ nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps, if (need_rbuf) { res = nxt_unit_get_mmap(ctx, pid, id); if (nxt_slow_path(res == NXT_UNIT_ERROR)) { - return NXT_UNIT_ERROR; + /* + * The caller releases rbuf on ERROR, so take it back off the + * wait queue first -- unless the segment arrived meanwhile and + * nxt_unit_incoming_mmap() already took it, in which case it + * is pending and still ours to wait on. + * + * Buffers other contexts parked behind this one did not ask + * for the segment themselves and nobody will ask for it now, + * so they go back to their contexts: the next read asks again. + */ + nxt_queue_init(&awaiting_rbuf); + + pthread_mutex_lock(&mmaps->mutex); + + if (mmaps->elts[id].hdr == NULL) { + nxt_queue_remove(&rbuf->link); + res = NXT_UNIT_ERROR; + + nxt_queue_add(&awaiting_rbuf, &mmaps->elts[id].awaiting_rbuf); + nxt_queue_init(&mmaps->elts[id].awaiting_rbuf); + + } else { + res = NXT_UNIT_AGAIN; + } + + pthread_mutex_unlock(&mmaps->mutex); + + if (res == NXT_UNIT_ERROR) { + nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + + nxt_unit_unpark_rbufs(ctx, &awaiting_rbuf); + } + + return res; } } @@ -4944,6 +5092,18 @@ nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps, } +/* + * The payload of an mmap message is an array of nxt_port_mmap_msg_t written + * by the router. It is walked with nxt_span_copy(): a payload that is not a + * whole number of records -- a partial tail -- is refused as a whole before + * anything is allocated, instead of its last "record" being read past the + * end of the message. Each record is copied out before use, and every + * field is bounds-checked before it reaches an index or pointer arithmetic: + * mmap_id by nxt_unit_mmap_at(), which refuses an id whose slot cannot be + * represented (the id is not capped otherwise -- see there), chunk_id and + * size by nxt_port_mmap_chunk_range_valid(). + */ + static int nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_read_buf_t *rbuf) @@ -4952,10 +5112,12 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, void *start; size_t nchunks; uint32_t size; + nxt_bool_t first; + nxt_span_t span; nxt_unit_impl_t *lib; nxt_unit_mmaps_t *mmaps; nxt_unit_mmap_buf_t *b, **incoming_tail; - nxt_port_mmap_msg_t *mmap_msg, *end; + nxt_port_mmap_msg_t mmap_msg; nxt_port_mmap_header_t *hdr; if (nxt_slow_path(recv_msg->size < sizeof(nxt_port_mmap_msg_t))) { @@ -4965,13 +5127,21 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, return NXT_UNIT_ERROR; } - mmap_msg = recv_msg->start; - end = nxt_pointer_to(recv_msg->start, recv_msg->size); + if (nxt_slow_path(recv_msg->size % sizeof(nxt_port_mmap_msg_t) != 0)) { + nxt_unit_alert(ctx, "#%"PRIu32": mmap_read: message size %d is not " + "a whole number of mmap records", + recv_msg->stream, (int) recv_msg->size); + + return NXT_UNIT_ERROR; + } incoming_tail = &recv_msg->incoming_buf; - /* Allocating buffer structures. */ - for (; mmap_msg < end; mmap_msg++) { + /* Allocating buffer structures, one per whole record. */ + for (size = 0; + size < recv_msg->size; + size += sizeof(nxt_port_mmap_msg_t)) + { b = nxt_unit_mmap_buf_get(ctx); if (nxt_slow_path(b == NULL)) { nxt_unit_warn(ctx, "#%"PRIu32": mmap_read: failed to allocate buf", @@ -4989,7 +5159,10 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, } b = recv_msg->incoming_buf; - mmap_msg = recv_msg->start; + first = 1; + + nxt_span_init(&span, recv_msg->start, + nxt_pointer_to(recv_msg->start, recv_msg->size)); lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -4997,9 +5170,10 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, pthread_mutex_lock(&mmaps->mutex); - for (; mmap_msg < end; mmap_msg++) { + while (nxt_span_copy(&span, &mmap_msg, sizeof(nxt_port_mmap_msg_t)) == 0) { + res = nxt_unit_check_rbuf_mmap(ctx, mmaps, - recv_msg->pid, mmap_msg->mmap_id, + recv_msg->pid, mmap_msg.mmap_id, &hdr, rbuf); if (nxt_slow_path(res != NXT_UNIT_OK)) { @@ -5015,39 +5189,26 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, * would point outside the mapped data area before they reach the * pointer arithmetic below. */ - if (nxt_slow_path(!nxt_port_mmap_chunk_range_valid(mmap_msg->chunk_id, - mmap_msg->size, + if (nxt_slow_path(!nxt_port_mmap_chunk_range_valid(mmap_msg.chunk_id, + mmap_msg.size, &nchunks))) { nxt_unit_alert(ctx, "#%"PRIu32": mmap_read: invalid mmap message: " "chunk_id %"PRIu32", size %"PRIu32 " (chunks %zu, max %d)", - recv_msg->stream, mmap_msg->chunk_id, - mmap_msg->size, nchunks, PORT_MMAP_CHUNK_COUNT); - - pthread_mutex_unlock(&mmaps->mutex); + recv_msg->stream, mmap_msg.chunk_id, + mmap_msg.size, nchunks, PORT_MMAP_CHUNK_COUNT); - /* - * Entries before the rejected one are already populated, and - * this message is dropped rather than retried, so their chunks - * have to be marked free as well: nxt_unit_mmap_buf_release() - * alone would recycle the wrappers and leave the chunks busy in - * the peer's segment for good. Entries not reached yet have a - * NULL hdr and are skipped by nxt_unit_free_outgoing_buf(). - */ - while (recv_msg->incoming_buf != NULL) { - nxt_unit_mmap_buf_free(recv_msg->incoming_buf); - } - - return NXT_UNIT_ERROR; + goto invalid; } - start = nxt_port_mmap_chunk_start(hdr, mmap_msg->chunk_id); - size = mmap_msg->size; + start = nxt_port_mmap_chunk_start(hdr, mmap_msg.chunk_id); + size = mmap_msg.size; - if (recv_msg->start == mmap_msg) { + if (first) { recv_msg->start = start; recv_msg->size = size; + first = 0; } b->buf.start = start; @@ -5061,13 +5222,31 @@ nxt_unit_mmap_read(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, recv_msg->stream, start, (int) size, (int) hdr->src_pid, (int) hdr->dst_pid, - (int) hdr->id, (int) mmap_msg->chunk_id, - (int) mmap_msg->size); + (int) hdr->id, (int) mmap_msg.chunk_id, + (int) mmap_msg.size); } pthread_mutex_unlock(&mmaps->mutex); return NXT_UNIT_OK; + +invalid: + + pthread_mutex_unlock(&mmaps->mutex); + + /* + * Entries before the rejected one are already populated, and this + * message is dropped rather than retried, so their chunks have to be + * marked free as well: nxt_unit_mmap_buf_release() alone would recycle + * the wrappers and leave the chunks busy in the peer's segment for good. + * Entries not reached yet have a NULL hdr and are skipped by + * nxt_unit_free_outgoing_buf(). + */ + while (recv_msg->incoming_buf != NULL) { + nxt_unit_mmap_buf_free(recv_msg->incoming_buf); + } + + return NXT_UNIT_ERROR; } diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 9ffc10c47..5ce647a37 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -430,5 +430,10 @@ void nxt_unit_test_ctx_set_ready(nxt_unit_ctx_t *ctx, uint8_t val); void nxt_unit_test_ctx_quit_graceful(nxt_unit_ctx_t *ctx); #endif +#if (NXT_TESTS || NXT_FUZZ_BUILD) +int nxt_unit_test_process_msg(nxt_unit_ctx_t *ctx, const void *msg, + size_t size, int fd); +#endif + #endif /* _NXT_UNIT_H_INCLUDED_ */ diff --git a/src/nxt_unit_sptr.h b/src/nxt_unit_sptr.h index 314416e4e..071391148 100644 --- a/src/nxt_unit_sptr.h +++ b/src/nxt_unit_sptr.h @@ -35,4 +35,53 @@ nxt_unit_sptr_get(nxt_unit_sptr_t *sptr) } +/* + * Resolves an sptr field of a peer-supplied buffer, returning the pointer + * when the [length]-byte range it addresses is wholly inside the buffer and + * NULL otherwise. Shared by both directions of the trust boundary: libunit + * vets every sptr of an nxt_unit_request_t before the application sees it + * (src/nxt_unit.c), and the router vets an application's nxt_unit_response_t + * (src/nxt_router.c). + * + * sptr->base aliases the address of the sptr itself (the union encodes an + * offset relative to that location), so this also implicitly checks that + * the sptr is inside the buffer. The offset is read once: the buffer may be + * shared memory the peer keeps writing, so it is read through volatile and + * the caller uses the returned pointer rather than resolving the sptr again. + */ +static inline void * +nxt_unit_sptr_in_buf(nxt_unit_sptr_t *sptr, uint32_t length, + void *buf_start, uint32_t buf_size) +{ + size_t sptr_off; + uint32_t offset; + + if ((uint8_t *) sptr < (uint8_t *) buf_start) { + return NULL; + } + + sptr_off = (uint8_t *) sptr - (uint8_t *) buf_start; + + /* + * The sptr itself must fit inside the buffer before its offset is read; + * the first test keeps the subtraction from underflowing. + */ + if (buf_size < sizeof(nxt_unit_sptr_t) + || sptr_off > buf_size - sizeof(nxt_unit_sptr_t)) + { + return NULL; + } + + offset = *(volatile uint32_t *) &sptr->offset; + + if (offset > buf_size - sptr_off + || length > buf_size - sptr_off - offset) + { + return NULL; + } + + return sptr->base + offset; +} + + #endif /* _NXT_UNIT_SPTR_H_INCLUDED_ */ diff --git a/src/test/nxt_port_mmap_read_test.c b/src/test/nxt_port_mmap_read_test.c new file mode 100644 index 000000000..1e16344ee --- /dev/null +++ b/src/test/nxt_port_mmap_read_test.c @@ -0,0 +1,327 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * nxt_port_mmap_read() (src/nxt_port_memory.c) turns an untrusted peer's + * array of nxt_port_mmap_msg_t into buffers over its shared memory. It + * used to read a partial last record past mem.free. The records sit at the + * end of a page followed by a PROT_NONE page, so an overrun faults in every + * build; each case runs in a child. The other rows pin the per-field checks. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +typedef struct { + const char *name; + nxt_pid_t pid_off; /* 1: an unknown sender */ + nxt_uint_t nrecords; + size_t tail; + nxt_uint_t expect_bufs; + nxt_port_mmap_msg_t records[2]; +} nxt_mmap_read_case_t; + + +static const nxt_mmap_read_case_t nxt_mmap_read_cases[] = { + { "one record", 0, 1, 0, 1, { { 0, 0, 100 } } }, + { "two records", 0, 2, 0, 2, + { { 0, 0, 100 }, { 0, 1, PORT_MMAP_CHUNK_SIZE } } }, + { "partial tail only", 0, 0, 5, 0, { { 0 } } }, + { "unknown mmap_id", 0, 1, 0, 0, { { 1, 0, 100 } } }, + { "huge mmap_id", 0, 1, 0, 0, { { 0xFFFFFFFF, 0, 100 } } }, + { "chunk_id past the data area", 0, 1, 0, 0, + { { 0, PORT_MMAP_CHUNK_COUNT, 100 } } }, + { "size past the data area", 0, 1, 0, 0, + { { 0, PORT_MMAP_CHUNK_COUNT - 1, PORT_MMAP_CHUNK_SIZE + 1 } } }, + { "size near UINT32_MAX", 0, 1, 0, 0, { { 0, 0, 0xFFFFFFFF } } }, + { "unknown sender", 1, 1, 0, 0, { { 0, 0, 100 } } }, +}; + + +static nxt_task_t *nxt_mmap_read_task; +static nxt_port_t *nxt_mmap_read_port; + + +static int +nxt_mmap_read_child(void *data) +{ + u_char *page, *start, *end; + size_t psize, len; + nxt_uint_t i, nbufs; + nxt_buf_t *b, *in; + nxt_port_recv_msg_t msg; + const nxt_mmap_read_case_t *tc; + + tc = data; + psize = getpagesize(); + + page = mmap(NULL, 2 * psize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (page == MAP_FAILED || mprotect(page + psize, psize, PROT_NONE) != 0) { + return 1; + } + + len = tc->nrecords * sizeof(nxt_port_mmap_msg_t); + end = page + psize; + start = end - len - tc->tail; + + /* A zero tail reads as mmap_id 0, which exists, so the overrun is real. */ + nxt_memcpy(start, tc->records, len); + nxt_memzero(start + len, tc->tail); + + in = nxt_buf_mem_alloc(nxt_mmap_read_port->mem_pool, 0, 0); + if (in == NULL) { + return 1; + } + + in->mem.start = start; + in->mem.pos = start; + in->mem.free = end; + in->mem.end = end; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + msg.port = nxt_mmap_read_port; + msg.buf = in; + msg.port_msg.pid = nxt_pid + 41 + tc->pid_off; + msg.port_msg.mmap = 1; + + nxt_port_mmap_read(nxt_mmap_read_task, &msg); + + nbufs = 0; + + for (b = msg.buf; b != NULL && b != in; b = b->next) { + nbufs++; + } + + len = 0; + + for (i = 0; i < tc->expect_bufs; i++) { + len += tc->records[i].size; + } + + return nbufs != tc->expect_bufs || msg.size != len + || (size_t) (in->mem.pos - start) + != tc->expect_bufs * sizeof(nxt_port_mmap_msg_t); +} + + +nxt_int_t +nxt_port_mmap_read_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_buf_t *seg, *segs[3]; + nxt_uint_t i, nsegs; + nxt_bool_t incoming_mutex; + nxt_task_t *task; + nxt_process_t *process; + nxt_runtime_t *rt, *saved_rt; + nxt_chunk_id_t c; + nxt_event_engine_t engine, *saved_engine; + nxt_mmap_read_case_t tc; + nxt_port_mmap_handler_t *mmap_handler; + + nxt_thread_time_update(thr); + + task = thr->task; + task->thread = thr; + nxt_mmap_read_task = task; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + process = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + + if (nxt_slow_path(rt == NULL || process == NULL + || nxt_thread_mutex_create(&rt->processes_mutex) + != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + nxt_memzero(&engine, sizeof(engine)); + engine.mem_pool = mp; + engine.task.thread = thr; + engine.task.log = thr->log; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + + ret = NXT_ERROR; + nsegs = 0; + incoming_mutex = 0; + + process->pid = nxt_pid + 41; + process->use_count = 1; + nxt_queue_init(&process->ports); + + if (nxt_slow_path(nxt_thread_mutex_create(&process->incoming.mutex) + != NXT_OK)) + { + goto done; + } + + incoming_mutex = 1; + + nxt_runtime_process_add(task, process); + + /* The sender's segment 0, allocated through the same array. */ + seg = nxt_port_mmap_get_buf(task, &process->incoming, + 2 * PORT_MMAP_CHUNK_SIZE); + if (nxt_slow_path(seg == NULL)) { + goto done; + } + + segs[nsegs++] = seg; + + mmap_handler = seg->parent; + c = nxt_port_mmap_chunk_id(mmap_handler->hdr, seg->mem.pos); + + nxt_mmap_read_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(nxt_mmap_read_port == NULL)) { + goto done; + } + + nxt_mmap_read_port->pair[0] = -1; + nxt_mmap_read_port->pair[1] = -1; + nxt_mmap_read_port->socket.fd = -1; + + for (i = 0; i < nxt_nitems(nxt_mmap_read_cases); i++) { + if (nxt_test_in_child(thr, nxt_mmap_read_cases[i].name, + nxt_mmap_read_child, + (void *) &nxt_mmap_read_cases[i]) + != 0) + { + nxt_log_alert(thr->log, "port mmap read test \"%s\" failed", + nxt_mmap_read_cases[i].name); + goto done; + } + } + + /* A whole record followed by every possible partial tail. */ + + tc = nxt_mmap_read_cases[0]; + + for (tc.tail = 1; tc.tail < sizeof(nxt_port_mmap_msg_t); tc.tail++) { + if (nxt_test_in_child(thr, "partial tail", nxt_mmap_read_child, &tc) + != 0) + { + nxt_log_alert(thr->log, "port mmap read test: a %uz-byte tail " + "failed", tc.tail); + goto done; + } + } + + /* + * The peer maps the segment writable, so the busy sentinel past the + * last chunk may read as free: nxt_port_mmap_increase_buf() must still + * not grow a buffer past the data area. + */ + seg = nxt_port_mmap_get_buf(task, &process->incoming, + (PORT_MMAP_CHUNK_COUNT - c - 2) + * PORT_MMAP_CHUNK_SIZE); + if (seg != NULL) { + segs[nsegs++] = seg; + } + + if (nxt_slow_path(seg == NULL || seg->parent != mmap_handler)) { + goto done; + } + + for (i = 0; i < 4; i++) { + nxt_port_mmap_set_chunk_free(mmap_handler->hdr->free_map, + PORT_MMAP_CHUNK_COUNT + i); + } + + (void) nxt_port_mmap_increase_buf(task, seg, + nxt_buf_mem_free_size(&seg->mem) + + 4 * PORT_MMAP_CHUNK_SIZE, 1); + + if (seg->mem.end > nxt_port_mmap_chunk_start(mmap_handler->hdr, + PORT_MMAP_CHUNK_COUNT)) + { + nxt_log_alert(thr->log, "port mmap read test: increase_buf grew " + "past the data area"); + goto done; + } + + /* Nor may nxt_port_mmap_get() continue from the last chunk past it. */ + nxt_port_mmap_set_chunk_free(mmap_handler->hdr->free_map, + PORT_MMAP_CHUNK_COUNT - 1); + + seg = nxt_port_mmap_get_buf(task, &process->incoming, + 2 * PORT_MMAP_CHUNK_SIZE); + if (seg != NULL) { + segs[nsegs++] = seg; + } + + if (seg != NULL && seg->parent == mmap_handler + && seg->mem.end > nxt_port_mmap_chunk_start(mmap_handler->hdr, + PORT_MMAP_CHUNK_COUNT)) + { + nxt_log_alert(thr->log, "port mmap read test: get_buf ran past " + "the data area"); + goto done; + } + + ret = NXT_OK; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port mmap read test passed"); + +done: + + /* + * Each nxt_port_mmap_get_buf() took a reference on its segment's + * handler, and process->incoming holds one more. The buffers are + * never completed here, so their references are dropped by hand + * (nxt_port_mmap_handler_use() is private to nxt_port_memory.c); the + * array's reference, dropped last by nxt_port_mmaps_destroy(), then + * unmaps and frees each handler. The buffers themselves + * come from mp. The port has its own pool, freed by the last + * reference; it was never linked to a process. nxt_runtime_process_add() + * put the process in rt->processes, a malloc'd table outside mp. + */ + for (i = 0; i < nsegs; i++) { + mmap_handler = segs[i]->parent; + (void) nxt_atomic_fetch_add(&mmap_handler->use_count, -1); + } + + nxt_port_mmaps_destroy(&process->incoming, 1); + + if (nxt_mmap_read_port != NULL) { + nxt_port_use(task, nxt_mmap_read_port, -1); + nxt_mmap_read_port = NULL; + } + + if (process->registered) { + nxt_runtime_process_remove(rt, process); + } + + if (incoming_mutex) { + nxt_thread_mutex_destroy(&process->incoming.mutex); + } + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + nxt_mp_destroy(mp); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index c622a58e8..7fdbcc244 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -71,7 +71,7 @@ nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), static nxt_int_t (*const nxt_security_tests[])(nxt_thread_t *) = { - nxt_checked_test, + nxt_checked_test, nxt_port_mmap_read_test, }; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 0197eee4c..c05a9691f 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -75,6 +75,7 @@ nxt_int_t nxt_port_fail_test(nxt_thread_t *thr); nxt_int_t nxt_fd_event_change_test(nxt_thread_t *thr); nxt_int_t nxt_port_use_unless_zero_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); +nxt_int_t nxt_port_mmap_read_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); diff --git a/src/test/nxt_unit_msg_test.c b/src/test/nxt_unit_msg_test.c new file mode 100644 index 000000000..275bb57f5 --- /dev/null +++ b/src/test/nxt_unit_msg_test.c @@ -0,0 +1,567 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Malformed port messages at libunit's receive side (src/nxt_unit.c), fed + * through nxt_unit_test_process_msg(). nxt_unit_mmap_read() used to read a + * partial last record past the message, mmap_id 0xFFFFFFFF wrapped the + * lib->incoming index, and a segment of any size was accepted. Cases that + * could crash the old code run in a child; a signal is a failure. + */ + +#include "nxt_main.h" +#include "nxt_port_memory_int.h" +#include "nxt_port_queue.h" +#include "nxt_app_queue.h" +#include "nxt_unit.h" + +#include +#include +#include +#include +#include +#include +#include + + +static int nxt_unit_msg_test_failures; +static int nxt_unit_msg_test_send_fails; +static int nxt_unit_msg_test_quit_called; +static nxt_unit_ctx_t *nxt_unit_msg_test_ctx; +static nxt_unit_ctx_t *nxt_unit_msg_test_follower; + +static int nxt_unit_msg_test_send_records_to(nxt_unit_ctx_t *ctx, + const nxt_port_mmap_msg_t *records, size_t nrecords, size_t tail); + + +static void +nxt_unit_msg_test_assert(int cond, const char *name) +{ + if (cond) { + printf("unit msg test: %-50s passed\n", name); + + } else { + printf("unit msg test: %-50s FAILED\n", name); + + nxt_unit_msg_test_failures++; + } +} + + +static void +nxt_unit_msg_test_handler(nxt_unit_request_info_t *req) +{ + nxt_unit_request_done(req, NXT_UNIT_ERROR); +} + + +static ssize_t +nxt_unit_msg_test_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, + const void *buf, size_t buf_size, const void *oob, size_t oob_size) +{ + const nxt_port_msg_t *msg; + nxt_port_mmap_msg_t rec; + + msg = buf; + + /* + * While one context asks the router for a segment, another one reads + * a record for the same segment and parks behind it. + */ + if (msg->type == _NXT_PORT_MSG_GET_MMAP + && nxt_unit_msg_test_follower != NULL) + { + rec.mmap_id = ((const nxt_port_msg_get_mmap_t *) (msg + 1))->id; + rec.chunk_id = 0; + rec.size = 100; + + (void) nxt_unit_msg_test_send_records_to(nxt_unit_msg_test_follower, + &rec, 1, 0); + + nxt_unit_msg_test_follower = NULL; + } + + return nxt_unit_msg_test_send_fails ? -1 : (ssize_t) buf_size; +} + + +static ssize_t +nxt_unit_msg_test_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *buf, + size_t buf_size, void *oob, size_t *oob_size) +{ + *oob_size = 0; + + return 0; +} + + +static int +nxt_unit_msg_test_shm(size_t size) +{ + int fd; + +#if (NXT_HAVE_MEMFD_CREATE) + fd = syscall(SYS_memfd_create, "unit.msg.test", 0); +#else + char name[64]; + + snprintf(name, sizeof(name), "/unit.msg.test.%d", (int) getpid()); + + fd = shm_open(name, O_CREAT | O_EXCL | O_RDWR, 0600); + shm_unlink(name); +#endif + + if (fd == -1) { + perror("shm"); + return -1; + } + + if (ftruncate(fd, size) == -1) { + perror("ftruncate"); + close(fd); + return -1; + } + + return fd; +} + + +/* A segment the router would hand over, from this process to itself. */ +static int +nxt_unit_msg_test_send_segment(size_t size, uint32_t id) +{ + int fd; + nxt_port_msg_t msg; + nxt_port_mmap_header_t *hdr; + + fd = nxt_unit_msg_test_shm(size); + if (fd == -1) { + return -2; + } + + hdr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (hdr == MAP_FAILED) { + perror("mmap"); + close(fd); + return -2; + } + + hdr->id = id; + hdr->src_pid = getpid(); + hdr->dst_pid = getpid(); + + munmap(hdr, size); + + memset(&msg, 0, sizeof(msg)); + + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_MMAP; + msg.last = 1; + + return nxt_unit_test_process_msg(nxt_unit_msg_test_ctx, &msg, sizeof(msg), + fd); +} + + +/* RPC_READY with the mmap bit: the result is the mmap read's verdict. */ +static int +nxt_unit_msg_test_send_records_to(nxt_unit_ctx_t *ctx, + const nxt_port_mmap_msg_t *records, size_t nrecords, size_t tail) +{ + u_char buf[256]; + size_t size; + nxt_port_msg_t msg; + + memset(&msg, 0, sizeof(msg)); + + msg.stream = 7; + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_RPC_READY; + msg.last = 1; + msg.mmap = 1; + + size = nrecords * sizeof(nxt_port_mmap_msg_t); + + memcpy(buf, &msg, sizeof(msg)); + memcpy(buf + sizeof(msg), records, size); + memset(buf + sizeof(msg) + size, 0, tail); + + return nxt_unit_test_process_msg(ctx, buf, sizeof(msg) + size + tail, -1); +} + + +static int +nxt_unit_msg_test_send_records(const nxt_port_mmap_msg_t *records, + size_t nrecords, size_t tail) +{ + return nxt_unit_msg_test_send_records_to(nxt_unit_msg_test_ctx, records, + nrecords, tail); +} + + +/* The child cases exit with the return code + 10, to tell it from 1, 2. */ + +#define NXT_UNIT_MSG_TEST_RC(rc) ((rc) + 10) + + +/* Runs "fn" in a child: asserts that it exits with RC(expect). */ +static void +nxt_unit_msg_test_in_child(const char *name, int (*fn)(void *), void *data, + int expect) +{ + int status; + pid_t child; + + status = -1; + fflush(stdout); + + child = fork(); + + if (child == 0) { + _exit(fn(data) & 0xFF); + } + + if (child != -1 && waitpid(child, &status, 0) == child + && WIFSIGNALED(status)) + { + printf("unit msg test: child killed by signal %d\n", WTERMSIG(status)); + } + + nxt_unit_msg_test_assert(child != -1 && WIFEXITED(status) + && WEXITSTATUS(status) + == NXT_UNIT_MSG_TEST_RC(expect), name); +} + + +static int +nxt_unit_msg_test_records_case(void *data) +{ + return NXT_UNIT_MSG_TEST_RC(nxt_unit_msg_test_send_records(data, 1, 0)); +} + + +typedef struct { + const char *name; + size_t size; + uint32_t id; + int expect; +} nxt_unit_msg_test_segment_t; + + +static const nxt_unit_msg_test_segment_t segments[] = { + { "segment id 0xFFFFFFFF is refused", PORT_MMAP_SIZE, 0xFFFFFFFF, + NXT_UNIT_ERROR }, + { "segment id 1000 is accepted", PORT_MMAP_SIZE, 1000, NXT_UNIT_OK }, + { "segment shorter than PORT_MMAP_SIZE is refused", + PORT_MMAP_HEADER_SIZE, 1, NXT_UNIT_ERROR }, + { "segment longer than PORT_MMAP_SIZE is refused", + 2 * PORT_MMAP_SIZE, 1, NXT_UNIT_ERROR }, +}; + + +static const struct { + const char *name; + nxt_port_mmap_msg_t rec; +} bad_records[] = { + { "mmap_id 0xFFFFFFFF is refused", { 0xFFFFFFFF, 0, 100 } }, + { "chunk_id past the data area is refused", + { 0, PORT_MMAP_CHUNK_COUNT, 100 } }, + { "size past the data area is refused", + { 0, PORT_MMAP_CHUNK_COUNT - 1, PORT_MMAP_CHUNK_SIZE + 1 } }, +}; + + +static int +nxt_unit_msg_test_segment_case(void *data) +{ + const nxt_unit_msg_test_segment_t *seg = data; + + return NXT_UNIT_MSG_TEST_RC(nxt_unit_msg_test_send_segment(seg->size, + seg->id)); +} + + +static void +nxt_unit_msg_test_quit(nxt_unit_ctx_t *ctx) +{ + nxt_unit_msg_test_quit_called = 1; +} + + +/* + * A record for an unknown segment parks the read buffer and asks the router + * for the segment. When that cannot be sent, the buffer used to be freed + * still counted in wait_items, so a graceful quit never happened. + */ +static int +nxt_unit_msg_test_get_mmap_fail_case(void *data) +{ + int rc; + u_char buf[sizeof(nxt_port_msg_t) + 1]; + nxt_port_msg_t msg; + nxt_port_mmap_msg_t rec; + + rec.mmap_id = 5; + rec.chunk_id = 0; + rec.size = 100; + + nxt_unit_msg_test_send_fails = 1; + rc = nxt_unit_msg_test_send_records(&rec, 1, 0); + nxt_unit_msg_test_send_fails = 0; + + if (rc != NXT_UNIT_ERROR) { + return 1; + } + + memset(&msg, 0, sizeof(msg)); + + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_QUIT; + msg.last = 1; + + memcpy(buf, &msg, sizeof(msg)); + buf[sizeof(msg)] = NXT_PORT_QUIT_GRACEFUL; + + (void) nxt_unit_test_process_msg(nxt_unit_msg_test_ctx, buf, sizeof(buf), + -1); + + return nxt_unit_msg_test_quit_called ? NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK) + : 2; +} + + +/* + * Two contexts park on the same unknown segment; only the first asks the + * router for it. When that failed, the second buffer stayed parked with + * its wait counted, so the second context never quit gracefully. + */ +static int +nxt_unit_msg_test_get_mmap_follower_case(void *data) +{ + int rc; + u_char buf[sizeof(nxt_port_msg_t) + 1]; + nxt_port_msg_t msg; + nxt_unit_ctx_t *ctx2; + nxt_port_mmap_msg_t rec; + + ctx2 = nxt_unit_ctx_alloc(nxt_unit_msg_test_ctx, NULL); + if (ctx2 == NULL) { + return 1; + } + + rec.mmap_id = 6; + rec.chunk_id = 0; + rec.size = 100; + + nxt_unit_msg_test_follower = ctx2; + nxt_unit_msg_test_send_fails = 1; + + rc = nxt_unit_msg_test_send_records(&rec, 1, 0); + + if (rc != NXT_UNIT_ERROR || nxt_unit_msg_test_follower != NULL) { + return 2; + } + + memset(&msg, 0, sizeof(msg)); + + msg.pid = getpid(); + msg.type = _NXT_PORT_MSG_QUIT; + msg.last = 1; + + memcpy(buf, &msg, sizeof(msg)); + buf[sizeof(msg)] = NXT_PORT_QUIT_GRACEFUL; + + /* The follower reads its record again, asks, fails, and quits. */ + + (void) nxt_unit_test_process_msg(ctx2, buf, sizeof(buf), -1); + + return nxt_unit_msg_test_quit_called ? NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK) + : 3; +} + + +static int +nxt_unit_msg_test_count_maps(void) +{ + int n; + char line[512]; + FILE *f; + + f = fopen("/proc/self/maps", "r"); + if (f == NULL) { + return -1; + } + + n = 0; + + while (fgets(line, sizeof(line), f) != NULL) { + n += (strstr(line, "unit.msg.test") != NULL); + } + + fclose(f); + + return n; +} + + +/* A duplicate segment id must not leak a mapping; the first one stays. */ +static int +nxt_unit_msg_test_dup_id_case(void *data) +{ + int i, before, after; + nxt_port_mmap_msg_t rec; + + before = nxt_unit_msg_test_count_maps(); + + for (i = 0; i < 3; i++) { + if (nxt_unit_msg_test_send_segment(PORT_MMAP_SIZE, 0) != NXT_UNIT_OK) { + return 1; + } + } + + after = nxt_unit_msg_test_count_maps(); + + if (before < 0 || after != before) { + printf("unit msg test: %d segment mappings, was %d\n", after, before); + return 2; + } + + rec.mmap_id = 0; + rec.chunk_id = 0; + rec.size = 100; + + return NXT_UNIT_MSG_TEST_RC(nxt_unit_msg_test_send_records(&rec, 1, 0)); +} + + +int +main(void) +{ + int rc, ready[2], router[2], read[2], shared[2]; + char name[64]; + size_t tail, i; + nxt_unit_init_t init; + nxt_port_mmap_msg_t rec[2]; + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, ready) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, router) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, read) == -1 + || socketpair(AF_UNIX, SOCK_DGRAM, 0, shared) == -1) + { + perror("socketpair"); + return 1; + } + + memset(&init, 0, sizeof(init)); + + init.callbacks.request_handler = nxt_unit_msg_test_handler; + init.callbacks.port_send = nxt_unit_msg_test_send; + init.callbacks.port_recv = nxt_unit_msg_test_recv; + init.callbacks.quit = nxt_unit_msg_test_quit; + + init.ready_port.id.pid = getpid(); + init.ready_port.id.id = 1; + init.ready_port.in_fd = -1; + init.ready_port.out_fd = ready[0]; + + init.ready_stream = 1; + + init.router_port.id.pid = getpid(); + init.router_port.id.id = 2; + init.router_port.in_fd = -1; + init.router_port.out_fd = router[0]; + + init.read_port.id.pid = getpid(); + init.read_port.id.id = 3; + init.read_port.in_fd = read[0]; + init.read_port.out_fd = read[1]; + + init.shared_port_fd = shared[0]; + init.shared_queue_fd = nxt_unit_msg_test_shm(sizeof(nxt_app_queue_t)); + init.log_fd = STDERR_FILENO; + + if (init.shared_queue_fd == -1) { + return 1; + } + + nxt_unit_msg_test_ctx = nxt_unit_init(&init); + if (nxt_unit_msg_test_ctx == NULL) { + printf("unit msg test: nxt_unit_init() failed\n"); + return 1; + } + + /* Segment 0, the one every well-formed record below refers to. */ + + rc = nxt_unit_msg_test_send_segment(PORT_MMAP_SIZE, 0); + + nxt_unit_msg_test_assert(rc == NXT_UNIT_OK, "segment 0 is accepted"); + + if (rc != NXT_UNIT_OK) { + return 1; + } + + memset(rec, 0, sizeof(rec)); + + rec[0].size = 100; + rec[1].chunk_id = 1; + rec[1].size = PORT_MMAP_CHUNK_SIZE; + + nxt_unit_msg_test_assert( + nxt_unit_msg_test_send_records(rec, 1, 0) == NXT_UNIT_OK, + "one whole record is accepted"); + + nxt_unit_msg_test_assert( + nxt_unit_msg_test_send_records(rec, 2, 0) == NXT_UNIT_OK, + "two whole records are accepted"); + + for (tail = 1; tail < sizeof(nxt_port_mmap_msg_t); tail++) { + snprintf(name, sizeof(name), "record + %d-byte partial tail refused", + (int) tail); + + nxt_unit_msg_test_assert( + nxt_unit_msg_test_send_records(rec, 1, tail) == NXT_UNIT_ERROR, + name); + } + + nxt_unit_msg_test_assert( + nxt_unit_msg_test_send_records(rec, 0, 5) == NXT_UNIT_ERROR, + "a lone partial record is refused"); + + for (i = 0; i < nxt_nitems(bad_records); i++) { + nxt_unit_msg_test_in_child(bad_records[i].name, + nxt_unit_msg_test_records_case, + (void *) &bad_records[i].rec, + NXT_UNIT_ERROR); + } + + /* Not capped: the router's outgoing segments are unbounded (#172). */ + + for (i = 0; i < nxt_nitems(segments); i++) { + nxt_unit_msg_test_in_child(segments[i].name, + nxt_unit_msg_test_segment_case, + (void *) &segments[i], segments[i].expect); + } + + nxt_unit_msg_test_in_child("failed get_mmap does not block a graceful " + "quit", nxt_unit_msg_test_get_mmap_fail_case, + NULL, NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("failed get_mmap unparks the other contexts", + nxt_unit_msg_test_get_mmap_follower_case, + NULL, NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("duplicate segment id does not leak a mapping", + nxt_unit_msg_test_dup_id_case, NULL, + NXT_UNIT_OK); + + if (nxt_unit_msg_test_failures != 0) { + printf("unit msg test: %d failure(s)\n", nxt_unit_msg_test_failures); + return 1; + } + + printf("unit msg test: all passed\n"); + + return 0; +} From 9db0b0b57388f42922c5dd278d2470a3c77d72e7 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 00:47:40 +0200 Subject: [PATCH 296/391] proxy: parse the upstream Transfer-Encoding as a list of codings nxt_h1p_peer_transfer_encoding() accepted only the exact value "chunked". RFC 9112 Sect. 6.1 makes coding names case-insensitive and the value a comma-separated list. So "Chunked" or "gzip, chunked" was not seen as chunked. The body was then framed by Content-Length or by connection close, and the chunk-size lines and CRLFs reached the client as body bytes. The 502 for chunked together with Content-Length also fired only for lowercase "chunked", so "Chunked" with "Content-Length: 3" relayed three bytes of chunk framing. The value is now parsed as a list. Elements are split on commas. Spaces and tabs around an element are removed. Empty elements are ignored (RFC 9110 Sect. 5.6.1). Names are compared without case. Several Transfer-Encoding lines form one list. The proxy decodes only chunked, and it never forwards Transfer-Encoding to the client. So a response is chunked only when the whole list is one "chunked". Every other list fails the response with 502: an unknown coding, "gzip, chunked", "chunked, gzip", chunked twice, an empty value, or a coding with parameters. RFC 9112 Sect. 6.3 would read such a body until the connection closes. But the proxy cannot decode that body, and the client would get it in a coding that no header names. nginx also answers 502 here. Chunked together with Content-Length stays a 502, now in any letter case. Content-Length never frames a body that has a Transfer-Encoding. Both 502 cases log a warning. Both checks run after the branch that completes a bodyless final response, not before it. RFC 9112 Sect. 6.3 ends a response to HEAD, and any 204 or 304, at the first empty line whatever Content-Length and Transfer-Encoding say. There is no body to decode and no framing to disagree about, so neither 502 applies: such a response is relayed as bodyless, and the client keeps its connection. This also drops the chunked-plus-Content-Length 502 for those three, which fired above that branch before. Behaviour, by the upstream Transfer-Encoding: upstream response before after chunked decoded decoded Chunked, CHUNKED, " chunked ," chunk framing relayed decoded chunked + Content-Length 502 502 Chunked + Content-Length 3 bytes of framing 502 gzip, chunked / chunked, gzip raw bytes relayed 502 chunked twice / chunked;a=b raw bytes relayed 502 identity, gzip, unknown, empty raw bytes relayed 502 any of the above on HEAD/204/304 bodyless bodyless chunked + CL on HEAD/204/304 502 bodyless The proxy sends every upstream request with "Connection: close", so connection reuse does not change. test/test_proxy_te.py sends these responses from a plain Python upstream on port 7976, which test/fake_upstream/README.md now lists. Without the fix, 14 of the 20 cases fail. The bodyless cases pipeline a second request behind the first, as test_proxy_head.py does, so a dropped keep-alive fails them. Four of them reproduce the 502 the check ordering caused; the other two passed before and stay as guards. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- CHANGES | 14 ++ docs/changes.xml | 16 ++ src/nxt_h1proto.c | 115 +++++++++++- test/fake_upstream/README.md | 6 +- test/test_proxy_te.py | 336 +++++++++++++++++++++++++++++++++++ 5 files changed, 477 insertions(+), 10 deletions(-) create mode 100644 test/test_proxy_te.py diff --git a/CHANGES b/CHANGES index 8c827077b..ee4f60124 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,20 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: the proxy recognised an upstream Transfer-Encoding only when + its value was exactly "chunked". "Chunked" or "gzip, chunked" was + not recognised, so the body was framed by Content-Length or by + connection close, and the chunk framing reached the client as body + bytes. The value is now parsed as a list of codings, and coding + names are compared without case. One "chunked" is decoded. Any + other transfer coding, and chunked together with Content-Length, + fails the request with 502: the proxy cannot decode such a body, + and it never forwards Transfer-Encoding to the client. A response + to HEAD, and any 204 or 304, is exempt from both 502s: it ends at + the first empty line, so no Transfer-Encoding frames it. Before, + such a response gave 502 when it carried chunked together with + Content-Length. + *) Feature: the --hardening=[off|default|strict] configure option adds compiler and linker hardening flags. Each flag is probed, and a compiler that does not support a flag only loses that flag. The diff --git a/docs/changes.xml b/docs/changes.xml index 6031dc3f6..d6d5ada71 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,22 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +The proxy recognised an upstream Transfer-Encoding only when its value was +exactly "chunked". "Chunked" or "gzip, chunked" was not recognised, so the +body was framed by Content-Length or by connection close, and the chunk +framing reached the client as body bytes. The value is now parsed as a list +of codings, and coding names are compared without case. One "chunked" is +decoded. Any other transfer coding, and chunked together with +Content-Length, fails the request with 502: the proxy cannot decode such a +body, and it never forwards Transfer-Encoding to the client. A response to +HEAD, and any 204 or 304, is exempt from both 502s: it ends at the first empty +line, so no Transfer-Encoding frames it. Before, such a response gave 502 when +it carried chunked together with Content-Length. + + + the --hardening=[off|default|strict] configure option adds compiler and diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 713dec0cc..c6737fdb1 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -2892,10 +2892,7 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) h1p = peer->proto.h1; - if (h1p->chunked && r->resp.content_length != NULL) { - peer->status = NXT_HTTP_BAD_GATEWAY; - break; - } + h1p->chunked = (h1p->transfer_encoding == NXT_HTTP_TE_CHUNKED); /* * RFC 9112 Sect. 6.3: a response to HEAD, and any 204 or 304 response, @@ -2947,6 +2944,39 @@ nxt_h1p_peer_header_read_done(nxt_task_t *task, void *obj, void *data) return; } + /* + * See nxt_h1p_peer_transfer_encoding(). A Transfer-Encoding the + * proxy cannot decode fails the response with 502. With any + * Transfer-Encoding, Content-Length does not frame the body, and both + * together is an error (RFC 9112 Sect. 6.3). The upstream connection + * is closed after every response, so a framing error never reaches a + * reused connection. + * + * Both checks come after the bodyless branch above, not before it. A + * response to HEAD, and any 204 or 304, ends at the first empty line + * no matter what Content-Length and Transfer-Encoding say -- the same + * Sect. 6.3 -- so there is no body for the proxy to decode, and no + * framing for these two fields to disagree about. Such a response has + * already been completed above. Failing it with 502 would lose a + * response over a field that frames nothing: the proxy never reads a + * body there, whatever coding the upstream named. + */ + if (h1p->transfer_encoding == NXT_HTTP_TE_UNSUPPORTED) { + nxt_log(task, NXT_LOG_WARN, + "upstream sent unsupported Transfer-Encoding"); + + peer->status = NXT_HTTP_BAD_GATEWAY; + break; + } + + if (h1p->chunked && r->resp.content_length != NULL) { + nxt_log(task, NXT_LOG_WARN, "upstream sent both " + "Transfer-Encoding and Content-Length"); + + peer->status = NXT_HTTP_BAD_GATEWAY; + break; + } + if (h1p->chunked) { h1p->chunked_parse.mem_pool = c->mem_pool; @@ -3471,20 +3501,89 @@ nxt_h1p_peer_free(nxt_task_t *task, void *obj, void *data) } +/* + * Transfer-Encoding is a comma-separated list of transfer codings + * (RFC 9112 Sect. 6.1). Coding names are case-insensitive. "chunked" must + * be the last coding, and it must appear only once. Several + * Transfer-Encoding lines form one list, in order. So each line starts from + * the state that the previous lines left in h1p->transfer_encoding. + * + * The proxy decodes only chunked, and it never forwards Transfer-Encoding to + * the client. So the result is NXT_HTTP_TE_CHUNKED only when the whole list + * is one "chunked", in any letter case. Every other list is + * NXT_HTTP_TE_UNSUPPORTED: an unknown coding, "gzip, chunked", + * "chunked, gzip", chunked twice, an empty value, or a coding with + * parameters. nxt_h1p_peer_header_read_done() then fails the response with + * 502. RFC 9112 Sect. 6.3 would read such a body until the connection + * closes. But the proxy cannot decode that body, and the client would get it + * in a coding that no header names. + */ + static nxt_int_t nxt_h1p_peer_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data) { + u_char *p, *end, *start, *last; + nxt_bool_t empty; + nxt_h1proto_t *h1p; + nxt_http_te_t te; nxt_http_request_t *r; r = ctx; field->skip = 1; - if (field->value_length == 7 - && memcmp(field->value, "chunked", 7) == 0) - { - r->peer->proto.h1->chunked = 1; + h1p = r->peer->proto.h1; + te = h1p->transfer_encoding; + + p = field->value; + end = p + field->value_length; + empty = 1; + + while (p < end) { + start = p; + + while (p < end && *p != ',') { + p++; + } + + last = p; + + if (p < end) { + p++; /* Skip the comma. */ + } + + while (start < last && (*start == ' ' || *start == '\t')) { + start++; + } + + while (last > start && (last[-1] == ' ' || last[-1] == '\t')) { + last--; + } + + /* RFC 9110 Sect. 5.6.1: empty list elements are ignored. */ + if (start == last) { + continue; + } + + empty = 0; + + if (te == NXT_HTTP_TE_NONE + && last - start == nxt_length("chunked") + && nxt_memcasecmp(start, "chunked", nxt_length("chunked")) == 0) + { + te = NXT_HTTP_TE_CHUNKED; + + } else { + te = NXT_HTTP_TE_UNSUPPORTED; + } + } + + /* A field line must carry at least one transfer coding. */ + if (empty) { + te = NXT_HTTP_TE_UNSUPPORTED; } + h1p->transfer_encoding = te; + return NXT_OK; } diff --git a/test/fake_upstream/README.md b/test/fake_upstream/README.md index fe149113d..468fc9efe 100644 --- a/test/fake_upstream/README.md +++ b/test/fake_upstream/README.md @@ -76,6 +76,7 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: | Port | Token | Mode (CLI) | Rust handler | pytest | |------|-------|-----------|--------------|--------| +| 7976 | — | — (plain-Python upstream in the test file) | — | `test_proxy_te.py` (upstream Transfer-Encoding framing: `chunked` in any letter case, lists, several field lines, Transfer-Encoding together with Content-Length) | | 7977 | — | — (plain-Python upstream in the test file) | — | `test_proxy_interim.py` (the proxy drops upstream 1xx responses and sends the final one) | | 7978 | — | — (plain-Python upstream in the test file) | — | `test_proxy_head.py` (bodyless upstream response: header block, with or without trailing bytes, then close — not a `fake_upstream` mode, reserved here so the 79xx block stays in one registry) | | 7979 | `dup_cl` | `dup-cl` | `respond_dup_cl` | `test_proxy_dup_cl_keepalive_disabled` (duplicate upstream Content-Length disables downstream keepalive for a keep-alive client; complete body keeps its terminal chunk) — out of the 7983–7999 block below, which is exhausted; 7980–7982 are `fake_otlp`, so 7974–7979 is the free gap | @@ -98,8 +99,9 @@ single `grep` finds every side of a case. E.g. token `chunked_response`: > **7999 is not available here.** `test/test_proxy.py` and `test/test_proxy_chunked.py` > already bind `SERVER_PORT = 7999` for their own upstream, so this registry ends at -> 7998. New `fake_upstream` slots go to the 7974–7976 gap (7977 is taken by -> `test_proxy_interim.py` above, 7978–7979 are taken above, 7980–7982 are `fake_otlp`). +> 7998. New `fake_upstream` slots go to the 7974–7975 gap (7976 is taken by +> `test_proxy_te.py` and 7977 by `test_proxy_interim.py` above, 7978–7979 are +> taken above, 7980–7982 are `fake_otlp`). A test pins its port as a module constant referencing this table: diff --git a/test/test_proxy_te.py b/test/test_proxy_te.py new file mode 100644 index 000000000..4e3d3b3c8 --- /dev/null +++ b/test/test_proxy_te.py @@ -0,0 +1,336 @@ +"""How the proxy frames an upstream response by its Transfer-Encoding. + +nxt_h1p_peer_transfer_encoding() used to accept only the exact value +"chunked". RFC 9112 Sect. 6.1 makes coding names case-insensitive and the +value a comma-separated list. So "Chunked" or "gzip, chunked" was not seen +as chunked. The body was then framed by Content-Length or by connection +close, and the chunk framing reached the client as body bytes. + +The proxy decodes only chunked, and it never forwards Transfer-Encoding to +the client. So the whole list must be one "chunked", in any letter case. +Whitespace and empty list elements around it are allowed. Every other +Transfer-Encoding gives 502. Chunked together with Content-Length also +gives 502 (RFC 9112 Sect. 6.3). Content-Length never frames a body that +has a Transfer-Encoding. + +Neither 502 applies to a response the same Sect. 6.3 says has no body: a +response to HEAD, and any 204 or 304, ends at the first empty line whatever +Transfer-Encoding and Content-Length say. There is no body to decode there, +so those responses are completed as bodyless and relayed normally. The +keep-alive half of that path is pinned in test_proxy_head.py, whose pipeline +idiom the bodyless cases below reuse. + +The upstream is a plain Python socket server. The request target selects +the case. +""" + +import select +import socket + +import pytest + +from conftest import run_process +from unit.applications.proto import ApplicationProto +from unit.utils import waitforsocket + +client = ApplicationProto() + +# Reserved in test/fake_upstream/README.md's port registry. +UPSTREAM_PORT = 7976 + +BODY = 'hello, world' + +# A chunked body whose chunk-size lines and CRLFs would leak into the relayed +# body if the proxy did not decode it. +CHUNKED_BODY = '5\r\nhello\r\n7\r\n, world\r\n0\r\n\r\n' + + +def resp(*fields, body=CHUNKED_BODY): + head = ''.join(f'{f}\r\n' for f in fields) + return f'HTTP/1.1 200 OK\r\n{head}Connection: close\r\n\r\n{body}' + + +def bodyless(status, *fields): + """A response with no body at all, whatever its header fields claim.""" + + head = ''.join(f'{f}\r\n' for f in fields) + return f'HTTP/1.1 {status}\r\n{head}Connection: close\r\n\r\n' + + +UPSTREAM_RESPONSES = { + '/lower': resp('Transfer-Encoding: chunked'), + '/upper': resp('Transfer-Encoding: CHUNKED'), + '/mixed': resp('Transfer-Encoding: Chunked'), + '/ows': resp('Transfer-Encoding: ,\t chunked ,'), + # Content-Length would cut the body after the first chunk-size line. + '/mixed-cl': resp('Transfer-Encoding: Chunked', 'Content-Length: 3'), + '/lower-cl': resp('Transfer-Encoding: chunked', 'Content-Length: 3'), + '/gzip-chunked': resp('Transfer-Encoding: gzip, chunked'), + '/gzip-chunked-cl': resp( + 'Transfer-Encoding: gzip, Chunked', 'Content-Length: 3' + ), + '/chunked-gzip': resp('Transfer-Encoding: chunked, gzip'), + '/two-lines': resp( + 'Transfer-Encoding: gzip', 'Transfer-Encoding: chunked' + ), + '/chunked-twice': resp( + 'Transfer-Encoding: chunked', 'Transfer-Encoding: chunked' + ), + '/chunked-param': resp('Transfer-Encoding: chunked;a=b'), + '/empty': resp('Transfer-Encoding: ,'), + '/identity': resp( + 'Transfer-Encoding: identity', 'Content-Length: 3', body=BODY + ), + # Bodyless final responses carrying a Transfer-Encoding the proxy cannot + # decode. Every one of them writes only what is here and closes: none + # sends the body its own header names. + '/204-gzip': bodyless('204 No Content', 'Transfer-Encoding: gzip'), + '/304-identity': bodyless('304 Not Modified', 'Transfer-Encoding: identity'), + '/head-param': bodyless('200 OK', 'Transfer-Encoding: chunked;a=b'), + # Exact "chunked" reached the bodyless path before this change too, so this + # one guards against the move breaking it. + '/204-chunked': bodyless('204 No Content', 'Transfer-Encoding: chunked'), + # The other 502: chunked and Content-Length together. A 304 keeps its + # Content-Length (RFC 9110 Sect. 9.3.2 for HEAD, the cached length here). + '/304-chunked-cl': bodyless( + '304 Not Modified', 'Transfer-Encoding: chunked', 'Content-Length: 10' + ), + # The trailing request of every pipeline; this one does have a body. + '/ok': 'HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK', +} + + +def run_server(server_port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.bind(('127.0.0.1', server_port)) + sock.listen(10) + + def recv_request(conn): + data = b'' + + while b'\r\n\r\n' not in data: + if not select.select([conn], [], [], 5)[0]: + break + + part = conn.recv(4096) + if not part: + break + + data += part + + return data.decode('utf-8', errors='ignore') + + while True: + conn, _ = sock.accept() + + request = recv_request(conn) + target = request.split(' ')[1] if ' ' in request else '' + + conn.sendall( + UPSTREAM_RESPONSES.get( + target, 'HTTP/1.1 500 Internal Server Error\r\n\r\n' + ).encode() + ) + conn.close() + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + run_process(run_server, UPSTREAM_PORT) + waitforsocket(UPSTREAM_PORT) + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [ + {"action": {"proxy": f'http://127.0.0.1:{UPSTREAM_PORT}'}} + ], + } + ), 'proxy configuration' + + +def get(target): + return client.get(url=target, headers={'Host': 'localhost', + 'Connection': 'close'}) + + +@pytest.mark.parametrize('target', ['/lower', '/upper', '/mixed', '/ows']) +def test_proxy_te_chunked(target): + resp = get(target) + + assert resp['status'] == 200, 'status' + assert resp['body'] == BODY, 'chunked body decoded' + assert 'Content-Length' not in resp['headers'], 'no Content-Length' + + +@pytest.mark.parametrize( + 'target', + [ + '/mixed-cl', + '/lower-cl', + '/gzip-chunked', + '/gzip-chunked-cl', + '/chunked-gzip', + '/two-lines', + '/chunked-twice', + '/chunked-param', + '/empty', + '/identity', + ], +) +def test_proxy_te_bad_gateway(target): + resp = get(target) + + assert resp['status'] == 502, 'status' + assert 'hello' not in resp['body'], 'upstream body not relayed' + + +def pipeline(method, target): + """Send " " and a GET /ok back to back, read to EOF. + + Both requests leave in one write, so the second is already sitting in the + listener's read buffer when the first response is generated: it can only be + answered if the connection survives that response. That is how a + mis-framed bodyless response shows itself -- nxt_h1p_peer_closed() sets + r->truncated and r->inconsistent, and nxt_h1p_request_close() then drops + the client keep-alive. Copied from test_proxy_head.py, which pins the same + path without a Transfer-Encoding in play. + """ + + request = ( + f'{method} {target} HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + 'GET /ok HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + '\r\n' + ) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.settimeout(10) + + try: + sock.connect(('127.0.0.1', 8080)) + sock.sendall(request.encode()) + + data = b'' + while True: + try: + part = sock.recv(4096) + except socket.timeout: + break + + if not part: + break + + data += part + + finally: + sock.close() + + return data.decode('utf-8', errors='ignore') + + +def check_bodyless(raw, status): + """Assert the exact wire shape: bodyless header, then the whole GET /ok. + + Splitting on the header terminator pins both halves at once -- a 502 in + place of the first response, a body leaking out of it, or a missing second + response all change the number of parts or their content. + """ + + parts = raw.split('\r\n\r\n') + + assert len(parts) == 3, f'expected two responses, got {raw!r}' + + first, second, body = parts + + assert first.startswith( + f'HTTP/1.1 {status}' + ), f'first response status line: {first!r}' + assert ( + 'transfer-encoding' not in first.lower() + ), f'Transfer-Encoding relayed to the client: {first!r}' + assert ( + 'close' not in first.lower() + ), f'keep-alive dropped on a bodyless response: {first!r}' + + assert second.startswith( + 'HTTP/1.1 200' + ), f'pipelined request lost: {second!r}' + assert body == 'OK', f'pipelined response body: {body!r}' + + return first + + +def test_proxy_te_204_undecodable(): + """204 with "Transfer-Encoding: gzip": bodyless, not 502.""" + + first = check_bodyless(pipeline('GET', '/204-gzip'), 204) + + assert ( + 'content-length' not in first.lower() + ), f'Content-Length added to a 204: {first!r}' + + +def test_proxy_te_304_undecodable(): + """304 with "Transfer-Encoding: identity": bodyless, not 502.""" + + check_bodyless(pipeline('GET', '/304-identity'), 304) + + +def test_proxy_te_head_undecodable(): + """HEAD answered with "Transfer-Encoding: chunked;a=b": bodyless, not 502. + + A coding with parameters is not the bare "chunked" the proxy decodes, so + on any other status this is a 502 -- see test_proxy_te_bad_gateway's + /chunked-param. + """ + + check_bodyless(pipeline('HEAD', '/head-param'), 200) + + +def test_proxy_te_204_chunked(): + """204 with exact "chunked": unchanged by the check ordering. + + This case took the normal branch before the 502 checks moved and still + completes as bodyless, so it guards the move itself. + """ + + check_bodyless(pipeline('GET', '/204-chunked'), 204) + + +def test_proxy_te_304_chunked_content_length(): + """304 with chunked and Content-Length: bodyless, not 502. + + The chunked-plus-Content-Length 502 predates this change, and it fired + above the bodyless branch. RFC 9112 Sect. 6.3 ends this response at the + first empty line, so the two fields frame nothing and cannot disagree. + """ + + first = check_bodyless(pipeline('GET', '/304-chunked-cl'), 304) + + # RFC 9110 Sect. 9.3.2 / Sect. 15.4.5: a 304 reports the length the + # cached representation has. + assert ( + 'Content-Length: 10' in first + ), f'304 keeps Content-Length: {first!r}' + + +def test_proxy_te_bad_gateway_still_fires(): + """Control: an undecodable Transfer-Encoding on a status that has a body. + + test_proxy_te_bad_gateway covers the whole matrix; this pins the one shape + the bodyless cases above are the exemption from, over the same pipeline, + so the 502 and the exemption are read side by side. + """ + + raw = pipeline('GET', '/gzip-chunked') + + assert raw.startswith('HTTP/1.1 502'), f'status: {raw!r}' + assert 'hello' not in raw, f'upstream body not relayed: {raw!r}' From 23da8897173bfec4a362ba2d65192767e50e8676 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 00:14:30 +0000 Subject: [PATCH 297/391] fix: read what the application exchanges through shared memory once The request and the response travel between the router and an application in shared memory that the application processes map writable. The router wrote the request, but a sibling process of the same application can write it too. The application wrote the response, and can still write it while the router reads it. Both sides read values from that memory more than once, and followed offsets without a bounds check. Router: nxt_router_response_ready_handler() decoded the nxt_unit_response_t in place. fields_count was checked once and read again for the loop bound. The name, value and piggybacked body sptrs were resolved with nxt_unit_sptr_get() without a check, so an application could point the router anywhere. The decoding moves to nxt_router_response_header_parse(). Every value is read once, through volatile, into a local. Every sptr is resolved with nxt_unit_sptr_in_buf() against the buffer, the piggybacked body too. A response that does not pass is refused with an alert, as before. libunit: nxt_unit_process_req_headers() checked the request on arrival but read several values again afterwards: the preread body sptr, the lengths, fields_count and the cached field indexes. nxt_unit_request_group_dup_fields() read fields_count and every field again, later, from the handler. nxt_unit_process_websocket() read the two header bytes of a frame three times. Now the fixed part of the request and each field are copied out through volatile, the checks and the uses take the copies, and the pointers that nxt_unit_sptr_in_buf() returned are the ones used. group_dup_fields reads the count once, checks it against the request buffer again and resolves the names with nxt_unit_sptr_in_buf(). The websocket frame header is copied out once. What the application reads through req->request is still the segment. Copying the request out on arrival is a design change and is left for a separate discussion. This closes the double-fetch thread of the #445 review (fields_count read twice) and includes the libunit part that was #468. The request side, the length checks in nxt_router_prepare_msg(), is already in master (593f9f5c). Tests: src/test/nxt_router_response_parse_test.c drives the router parser with a well-formed buffer and one damaged value at a time: a short buffer, a count, a name, value or body sptr, and a length past the buffer. src/test/nxt_unit_msg_test.c writes a request into segment 0 the way the router does and sends REQ_HEADERS and WEBSOCKET messages about it; the cases that crashed the old code run in a child, and a thread that keeps changing the preread offset must not move the body read out of the buffer. fuzzing/nxt_router_app_response_fuzz.c is a libFuzzer target for the router parser, with a seed corpus, run in CI with the others. The libunit fuzz harness now answers a websocket handshake with an upgrade so that frames reach the frame parser. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/fuzzing.yml | 11 +- CHANGES | 9 + auto/fuzzing | 10 + auto/sources | 2 + docs/changes.xml | 12 + fuzzing/README.md | 20 +- fuzzing/build-fuzz.sh | 1 + .../no_fields.bin | Bin 0 -> 24 bytes .../piggyback.bin | Bin 0 -> 35 bytes .../valid_simple.bin | Bin 0 -> 62 bytes .../req_headers_ws_handshake.bin | Bin 0 -> 212 bytes .../websocket_masked_ext.bin | Bin 0 -> 41 bytes fuzzing/nxt_router_app_response_fuzz.c | 95 +++ fuzzing/nxt_unit_msg_fuzz.c | 44 +- fuzzing/run-ci.sh | 7 +- src/nxt_router.c | 201 +++--- src/nxt_router.h | 4 + src/nxt_unit.c | 223 ++++--- src/test/nxt_router_response_parse_test.c | 197 ++++++ src/test/nxt_tests.c | 1 + src/test/nxt_tests.h | 1 + src/test/nxt_unit_msg_test.c | 573 +++++++++++++++++- 22 files changed, 1248 insertions(+), 163 deletions(-) create mode 100644 fuzzing/fuzz_router_app_response_seed_corpus/no_fields.bin create mode 100644 fuzzing/fuzz_router_app_response_seed_corpus/piggyback.bin create mode 100644 fuzzing/fuzz_router_app_response_seed_corpus/valid_simple.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/req_headers_ws_handshake.bin create mode 100644 fuzzing/fuzz_unit_msg_seed_corpus/websocket_masked_ext.bin create mode 100644 fuzzing/nxt_router_app_response_fuzz.c create mode 100644 src/test/nxt_router_response_parse_test.c diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index cc5ec5b0b..b875a97c2 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -17,6 +17,7 @@ on: - 'src/nxt_unit*' - 'src/nxt_port_memory_int.h' - 'src/nxt_span.h' + - 'src/nxt_router*' # configure --fuzz, auto/fuzzing and auto/sources are what produce the # binaries; break them and the job has nothing to run. - configure @@ -30,7 +31,7 @@ on: required: false default: '60' targets: - description: 'Space-separated fuzzer names (empty = all six)' + description: 'Space-separated fuzzer names (empty = all seven)' required: false default: '' @@ -49,10 +50,10 @@ jobs: libfuzzer: name: libFuzzer (ASan + UBSan) runs-on: ubuntu-latest - # Six targets at a 60s budget is a little over six minutes -- each + # Seven targets at a 60s budget is a little over seven minutes -- each # finishes within a second or two of its budget, and run-ci.sh caps the # worst case at two minutes each. The clang/ASan build is a few minutes - # on top. A dispatch can ask for a much longer budget across all six + # on top. A dispatch can ask for a much longer budget across all seven # targets, so it gets a much longer bound; a pull request must not, or a # hung job holds a runner for an hour. timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 120 || 25 }} @@ -86,11 +87,11 @@ jobs: ./configure --no-regex --no-pcre2 --fuzz=-fsanitize=fuzzer make fuzz -j"$(nproc)" - # All six targets, on a pull request and on a dispatch alike. Picking a + # All seven targets, on a pull request and on a dispatch alike. Picking a # subset per changed path was tried and dropped: every path filter above # already maps to a target, `fuzzing/**` maps to all of them, and a # target left out of the list is a harness that CI compiles but never - # runs. At about 61 seconds each the whole set is close to six minutes. + # runs. At about 61 seconds each the whole set is close to seven minutes. - name: Run the fuzzers env: FUZZ_SECONDS: ${{ github.event.inputs.seconds || '60' }} diff --git a/CHANGES b/CHANGES index ee4f60124..0c2bff6ce 100644 --- a/CHANGES +++ b/CHANGES @@ -92,6 +92,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 it. A request that waited for capacity previously had no deadline at all. The default remains no timeout. + *) Security: the router decoded the response an application returned + in shared memory without checking it: the field count was read + twice, and the name, value and body offsets were followed without a + bounds check. An application could point the router anywhere. + libunit read the request from shared memory more than once after + checking it. A sibling process of the same application, which maps + the same segment, could change a value in between. Both sides now + read every value once and check every offset against the buffer. + *) Security: the string form of "access_log" "format" wrote what a variable expanded to without escaping it. "$uri" is the percent-decoded target, so a request for "/%0d%0a..." put a real diff --git a/auto/fuzzing b/auto/fuzzing index ffc2b643b..590ca7948 100644 --- a/auto/fuzzing +++ b/auto/fuzzing @@ -24,6 +24,7 @@ fuzz: $NXT_BUILD_DIR/fuzz_basic \\ $NXT_BUILD_DIR/fuzz_http_h1p \\ $NXT_BUILD_DIR/fuzz_http_h1p_peer \\ $NXT_BUILD_DIR/fuzz_json \\ + $NXT_BUILD_DIR/fuzz_router_app_response \\ $NXT_BUILD_DIR/fuzz_unit_msg $NXT_BUILD_DIR/fuzz_basic: \$(NXT_FUZZ_OBJS) \\ @@ -71,6 +72,15 @@ $NXT_BUILD_DIR/fuzz_json: \$(NXT_FUZZ_OBJS) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ $NXT_FUZZ +$NXT_BUILD_DIR/fuzz_router_app_response: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_router_app_response \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_router_app_response_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + # libunit's receive side: linked against libunit, not the core library. $NXT_BUILD_DIR/fuzz_unit_msg: \$(NXT_FUZZ_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC diff --git a/auto/sources b/auto/sources index d79948026..ac5185d92 100644 --- a/auto/sources +++ b/auto/sources @@ -210,6 +210,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_checked_test.c \ src/test/nxt_conn_close_test.c \ src/test/nxt_router_prepare_msg_test.c \ + src/test/nxt_router_response_parse_test.c \ " @@ -356,6 +357,7 @@ NXT_FUZZ_SRCS=" \ fuzzing/nxt_http_h1p_fuzz.c \ fuzzing/nxt_http_h1p_peer_fuzz.c \ fuzzing/nxt_json_fuzz.c \ + fuzzing/nxt_router_app_response_fuzz.c \ fuzzing/nxt_unit_msg_fuzz.c \ " diff --git a/docs/changes.xml b/docs/changes.xml index d6d5ada71..7fa6fc70f 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -136,6 +136,18 @@ remains no timeout. + + +the router decoded the response an application returned in shared memory +without checking it: the field count was read twice, and the name, value and +body offsets were followed without a bounds check. An application could point +the router anywhere. libunit read the request from shared memory more than +once after checking it. A sibling process of the same application, which maps +the same segment, could change a value in between. Both sides now read every +value once and check every offset against the buffer. + + + the string form of "access_log" "format" wrote what a variable expanded to diff --git a/fuzzing/README.md b/fuzzing/README.md index 090860c33..1a2813d68 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -42,6 +42,7 @@ $ mkdir -p build/fuzz_http_controller_seed $ mkdir -p build/fuzz_http_h1p_seed $ mkdir -p build/fuzz_http_h1p_peer_seed $ mkdir -p build/fuzz_json_seed +$ mkdir -p build/fuzz_router_app_response_seed $ mkdir -p build/fuzz_unit_msg_seed $ ./build/fuzz_basic build/fuzz_basic_seed fuzzing/fuzz_basic_seed_corpus @@ -49,6 +50,7 @@ $ ./build/fuzz_http_controller build/fuzz_http_controller_seed fuzzing/fuzz_ht $ ./build/fuzz_http_h1p build/fuzz_http_h1p_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_http_h1p_peer build/fuzz_http_h1p_peer_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_json build/fuzz_json_seed fuzzing/fuzz_json_seed_corpus +$ ./build/fuzz_router_app_response build/fuzz_router_app_response_seed fuzzing/fuzz_router_app_response_seed_corpus $ ./build/fuzz_unit_msg build/fuzz_unit_msg_seed fuzzing/fuzz_unit_msg_seed_corpus ``` @@ -74,6 +76,13 @@ the core library: it feeds one router-to-application port message through `nxt_unit_process_msg()` and the request handler. Its input format and what the harness steers are described at the top of the file. +`fuzz_router_app_response` (`nxt_router_app_response_fuzz.c`) is the other +side of that trust boundary: the router decoding an `nxt_unit_response_t` an +application handed back, through `nxt_router_response_header_parse()` in +`src/nxt_router.c` (extracted from `nxt_router_response_ready_handler()` so +it can be driven without a port message or a live application). The fuzz +input *is* the response buffer, byte for byte -- no framing on top. + ## In CI `.github/workflows/fuzzing.yml` builds these targets with clang, ASan and @@ -81,15 +90,16 @@ UBSan and runs `fuzzing/run-ci.sh`, which gives each target a bounded libFuzzer budget against its seed corpus and fails on the first crash. ```shell -$ fuzzing/run-ci.sh # all six, 60s each +$ fuzzing/run-ci.sh # all seven, 60s each $ fuzzing/run-ci.sh -t 120 fuzz_http_h1p # one target, longer ``` A pull request that touches `src/nxt_http*`, `src/nxt_h1proto*`, -`src/nxt_controller*`, `src/nxt_conf*`, `src/nxt_unit*` or `fuzzing/` runs -all six targets -- roughly six minutes; a manual `workflow_dispatch` takes a -budget and a target list as inputs. Reproducers land in -`build/fuzz-artifacts/` and are uploaded as an artifact when the job fails. +`src/nxt_controller*`, `src/nxt_conf*`, `src/nxt_unit*`, `src/nxt_router*` or +`fuzzing/` runs all seven targets -- roughly seven minutes; a manual +`workflow_dispatch` takes a budget and a target list as inputs. Reproducers +land in `build/fuzz-artifacts/` and are uploaded as an artifact when the job +fails. Each target finishes within a second or two of its budget. `run-ci.sh` caps it at the budget plus a minute anyway -- libFuzzer only checks the budget diff --git a/fuzzing/build-fuzz.sh b/fuzzing/build-fuzz.sh index a1bee2b3a..7f11e256b 100644 --- a/fuzzing/build-fuzz.sh +++ b/fuzzing/build-fuzz.sh @@ -14,6 +14,7 @@ mkdir -p build/fuzz_http_controller_seed mkdir -p build/fuzz_http_h1p_seed mkdir -p build/fuzz_http_h1p_peer_seed mkdir -p build/fuzz_json_seed +mkdir -p build/fuzz_router_app_response_seed mkdir -p build/fuzz_unit_msg_seed echo "" diff --git a/fuzzing/fuzz_router_app_response_seed_corpus/no_fields.bin b/fuzzing/fuzz_router_app_response_seed_corpus/no_fields.bin new file mode 100644 index 0000000000000000000000000000000000000000..8443c13f6aea70eab1fc3646f241825bb796657a GIT binary patch literal 24 OcmZQzKm}76A$$M?zW|m1 literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_router_app_response_seed_corpus/piggyback.bin b/fuzzing/fuzz_router_app_response_seed_corpus/piggyback.bin new file mode 100644 index 0000000000000000000000000000000000000000..1885f91142610c20d656f7416c5f693ba3ca22bc GIT binary patch literal 35 dcmZQzKmcwa^8^sH0C7fYPENi;d45q&3IHlJ1vCHv literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_router_app_response_seed_corpus/valid_simple.bin b/fuzzing/fuzz_router_app_response_seed_corpus/valid_simple.bin new file mode 100644 index 0000000000000000000000000000000000000000..f670b89542594f869e9de58d33b7551a009a5cdd GIT binary patch literal 62 xcmZQzfB;4)eF93?#xwA60ofctEC9sL`FSO&c_q3bl?ACKsTC#q1v!bCc>t|L3Wfjx literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_ws_handshake.bin b/fuzzing/fuzz_unit_msg_seed_corpus/req_headers_ws_handshake.bin new file mode 100644 index 0000000000000000000000000000000000000000..53889aaf3909d7ea4b1a2d7f235cd53442919728 GIT binary patch literal 212 zcmb1P;9+23U<6}vMi31H+kgya4o*%Mpe!el%?89wATb~Ysr!!xfI@I!0c3%&Ef6~c zu_q9F0dW8j`v9>w5Jv&Ai4X$|$ZUHcb^u~`*ANDekdOd + +/* DO NOT TRY THIS AT HOME! */ +#include "nxt_router.c" + + +#define KMININPUTLENGTH sizeof(nxt_unit_response_t) +#define KMAXINPUTLENGTH (64 * 1024) + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +extern char **environ; + + +/* A bad response is answered with nxt_alert(), which no log level silences. */ +static void nxt_cdecl +nxt_fuzz_log_handler(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...) +{ +} + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + if (nxt_lib_start("fuzzing", NULL, &environ) != NXT_OK) { + return NXT_ERROR; + } + + nxt_main_log.handler = nxt_fuzz_log_handler; + + if (nxt_http_response_hash_init(NULL) != NXT_OK) { + return NXT_ERROR; + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + nxt_mp_t *mp; + u_char *buf; + nxt_buf_t b; + nxt_http_request_t r; + + if (size < KMININPUTLENGTH || size > KMAXINPUTLENGTH) { + return 0; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return 0; + } + + /* Exactly the fuzzer's bytes, so ASan catches a sptr resolved outside. */ + buf = nxt_mp_alloc(mp, size); + if (buf == NULL) { + goto failed; + } + + memcpy(buf, data, size); + + nxt_memzero(&b, sizeof(nxt_buf_t)); + b.mem.start = buf; + b.mem.pos = buf; + b.mem.free = buf + size; + b.mem.end = buf + size; + + nxt_memzero(&r, sizeof(nxt_http_request_t)); + r.mem_pool = mp; + r.task.log = &nxt_main_log; + + (void) nxt_router_response_header_parse(&r.task, &r, &b); + +failed: + + nxt_mp_destroy(mp); + + return 0; +} diff --git a/fuzzing/nxt_unit_msg_fuzz.c b/fuzzing/nxt_unit_msg_fuzz.c index 9c8ed25b5..04e10225d 100644 --- a/fuzzing/nxt_unit_msg_fuzz.c +++ b/fuzzing/nxt_unit_msg_fuzz.c @@ -8,7 +8,9 @@ * walk (nxt_unit_mmap_read()), request arrival validation * (nxt_unit_process_req_headers(), nxt_unit_sptr_in_buf()), request body * and websocket frames -- and, for a request that comes out ready, the - * application's request handler, which reads every field and finishes it. + * application's request handler, which reads every field and finishes it, + * or upgrades it when it is a websocket handshake so that frames on the + * same stream reach the frame parser (nxt_unit_process_websocket()). * * Input layout: * @@ -56,6 +58,7 @@ #include #include #include +#include #include #include @@ -140,6 +143,43 @@ nxt_unit_msg_fuzz_handler(nxt_unit_request_info_t *req) nxt_unit_msg_fuzz_sink = sum; + /* + * A handshake is answered with an upgrade, as a websocket application + * would: the request then waits in the hash for WEBSOCKET messages on + * its stream, which is the only way a frame reaches the frame parser. + * The close frame (or the reset) finishes it. + */ + if (r->websocket_handshake + && nxt_unit_response_init(req, 101, 0, 0) == NXT_UNIT_OK + && nxt_unit_response_upgrade(req) == NXT_UNIT_OK + && nxt_unit_response_send(req) == NXT_UNIT_OK) + { + return; + } + + nxt_unit_request_done(req, NXT_UNIT_ERROR); +} + + +static void +nxt_unit_msg_fuzz_websocket(nxt_unit_websocket_frame_t *ws) +{ + u_char buf[64]; + ssize_t n; + + /* Reads the payload through the mask, as an application would. */ + n = nxt_unit_websocket_read(ws, buf, sizeof(buf)); + + nxt_unit_msg_fuzz_sink = ws->payload_len + ws->header->opcode + + (n > 0 ? buf[n - 1] : 0); + + nxt_unit_websocket_done(ws); +} + + +static void +nxt_unit_msg_fuzz_close(nxt_unit_request_info_t *req) +{ nxt_unit_request_done(req, NXT_UNIT_ERROR); } @@ -206,6 +246,8 @@ nxt_unit_msg_fuzz_setup(void) memset(&init, 0, sizeof(init)); init.callbacks.request_handler = nxt_unit_msg_fuzz_handler; + init.callbacks.websocket_handler = nxt_unit_msg_fuzz_websocket; + init.callbacks.close_handler = nxt_unit_msg_fuzz_close; init.callbacks.port_send = nxt_unit_msg_fuzz_send; init.callbacks.port_recv = nxt_unit_msg_fuzz_recv; diff --git a/fuzzing/run-ci.sh b/fuzzing/run-ci.sh index eeabfd7fb..38b6f6e0f 100755 --- a/fuzzing/run-ci.sh +++ b/fuzzing/run-ci.sh @@ -10,7 +10,7 @@ # fuzzing/run-ci.sh [-t SECONDS] [TARGET ...] # # -t SECONDS libFuzzer budget per target (default 60) -# TARGET ... fuzzer names; default is all six +# TARGET ... fuzzer names; default is all seven # # Expects the fuzzers to be built already: # CC=clang CXX=clang++ \ @@ -48,6 +48,8 @@ corpus_for() { fuzz_basic) echo fuzz_basic_seed_corpus ;; fuzz_json) echo fuzz_json_seed_corpus ;; fuzz_unit_msg) echo fuzz_unit_msg_seed_corpus ;; + fuzz_router_app_response) + echo fuzz_router_app_response_seed_corpus ;; fuzz_http_*) echo fuzz_http_seed_corpus ;; *) die "unknown target: $1" ;; esac @@ -61,7 +63,8 @@ dict_for() { } [ $# -gt 0 ] || set -- fuzz_basic fuzz_json fuzz_http_controller \ - fuzz_http_h1p fuzz_http_h1p_peer fuzz_unit_msg + fuzz_http_h1p fuzz_http_h1p_peer \ + fuzz_router_app_response fuzz_unit_msg # Freeze the list: the loop below rebuilds "$@" to hold each target's libFuzzer # arguments, so it cannot also be the list being iterated. diff --git a/src/nxt_router.c b/src/nxt_router.c index dd6afdfc7..bee06522d 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5339,20 +5339,144 @@ nxt_router_thread_exit_handler(nxt_task_t *task, void *obj, void *data) } +/* + * Decodes the nxt_unit_response_t that an application put into "b". The + * buffer is shared memory the application can still write, so every value + * is read once, through volatile, into a local and checked before use: the + * field count against the buffer, each sptr with nxt_unit_sptr_in_buf(). + * It is a separate function so that a test and the fuzz target + * (fuzzing/nxt_router_app_response_fuzz.c) can call it with a bare request. + */ + +static nxt_int_t +nxt_router_response_header_parse(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *b) +{ + size_t b_size, count; + u_char *p; + uint32_t piggyback_length; + nxt_int_t ret; + nxt_unit_field_t *f, uf; + nxt_http_field_t *field; + nxt_unit_response_t *resp; + + b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; + + if (nxt_slow_path(b_size < sizeof(nxt_unit_response_t) + || b_size > UINT32_MAX)) + { + nxt_alert(task, "response buffer too small: %z", b_size); + return NXT_ERROR; + } + + resp = (void *) b->mem.pos; + count = *(volatile uint32_t *) &resp->fields_count; + + if (nxt_slow_path(count > (b_size - sizeof(nxt_unit_response_t)) + / sizeof(nxt_unit_field_t))) + { + nxt_alert(task, "response buffer too small for fields count: %uz", + count); + return NXT_ERROR; + } + + for (f = resp->fields; f < resp->fields + count; f++) { + uf = *(volatile nxt_unit_field_t *) f; + + if (uf.skip) { + continue; + } + + field = nxt_http_resp_field_add(&r->resp, r->mem_pool); + + if (nxt_slow_path(field == NULL)) { + return NXT_ERROR; + } + + field->hash = uf.hash; + field->skip = 0; + field->hopbyhop = 0; + + field->name_length = uf.name_length; + field->value_length = uf.value_length; + field->name = nxt_unit_sptr_in_buf(&f->name, uf.name_length, + b->mem.pos, b_size); + field->value = nxt_unit_sptr_in_buf(&f->value, uf.value_length, + b->mem.pos, b_size); + + if (nxt_slow_path(field->name == NULL || field->value == NULL)) { + nxt_alert(task, "response field sptr out of bounds"); + return NXT_ERROR; + } + + ret = nxt_http_field_process(field, &nxt_response_fields_hash, r); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + nxt_debug(task, "header%s: %*s: %*s", + (field->skip ? " skipped" : ""), + (size_t) field->name_length, field->name, + (size_t) field->value_length, field->value); + + if (field->skip) { + if (r->resp.num_inline_fields > 0 + && field == &r->resp.inline_fields[r->resp.num_inline_fields - 1]) + { + r->resp.num_inline_fields--; + } else if (r->resp.fields != NULL && r->resp.fields->last != NULL) { + r->resp.fields->last->nelts--; + } + } + } + + r->status = *(volatile uint16_t *) &resp->status; + + piggyback_length = *(volatile uint32_t *) &resp->piggyback_content_length; + + if (piggyback_length != 0) { + p = nxt_unit_sptr_in_buf(&resp->piggyback_content, piggyback_length, + b->mem.pos, b_size); + if (nxt_slow_path(p == NULL)) { + nxt_alert(task, "response piggyback content out of bounds"); + return NXT_ERROR; + } + + b->mem.pos = p; + b->mem.free = p + piggyback_length; + + } else { + b->mem.pos = b->mem.free; + } + + return NXT_OK; +} + + +#if (NXT_TESTS) + +/* For src/test/nxt_router_response_parse_test.c. */ + +nxt_int_t +nxt_router_test_response_header_parse(nxt_task_t *task, + nxt_http_request_t *r, nxt_buf_t *b) +{ + return nxt_router_response_header_parse(task, r, b); +} + +#endif + + static void nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - size_t b_size, count; nxt_int_t ret; nxt_app_t *app; nxt_buf_t *b, *next, *out, *last_b, *owned_b; nxt_port_t *app_port; - nxt_unit_field_t *f; - nxt_http_field_t *field; nxt_http_status_t status; nxt_http_request_t *r; - nxt_unit_response_t *resp; nxt_request_rpc_data_t *req_rpc_data; /* What "fail:" answers with, unless a branch below knows better. */ @@ -5459,76 +5583,11 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_http_request_send_body(task, r, NULL); } else { - b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; - - if (nxt_slow_path(b_size < sizeof(nxt_unit_response_t))) { - nxt_alert(task, "response buffer too small: %z", b_size); - goto fail; - } - - resp = (void *) b->mem.pos; - count = (b_size - sizeof(nxt_unit_response_t)) - / sizeof(nxt_unit_field_t); - - if (nxt_slow_path(count < resp->fields_count)) { - nxt_alert(task, "response buffer too small for fields count: %D", - resp->fields_count); + ret = nxt_router_response_header_parse(task, r, b); + if (nxt_slow_path(ret != NXT_OK)) { goto fail; } - field = NULL; - - for (f = resp->fields; f < resp->fields + resp->fields_count; f++) { - if (f->skip) { - continue; - } - - field = nxt_http_resp_field_add(&r->resp, r->mem_pool); - - if (nxt_slow_path(field == NULL)) { - goto fail; - } - - field->hash = f->hash; - field->skip = 0; - field->hopbyhop = 0; - - field->name_length = f->name_length; - field->value_length = f->value_length; - field->name = nxt_unit_sptr_get(&f->name); - field->value = nxt_unit_sptr_get(&f->value); - - ret = nxt_http_field_process(field, &nxt_response_fields_hash, r); - if (nxt_slow_path(ret != NXT_OK)) { - goto fail; - } - - nxt_debug(task, "header%s: %*s: %*s", - (field->skip ? " skipped" : ""), - (size_t) field->name_length, field->name, - (size_t) field->value_length, field->value); - - if (field->skip) { - if (r->resp.num_inline_fields > 0 - && field == &r->resp.inline_fields[r->resp.num_inline_fields - 1]) - { - r->resp.num_inline_fields--; - } else if (r->resp.fields != NULL && r->resp.fields->last != NULL) { - r->resp.fields->last->nelts--; - } - } - } - - r->status = resp->status; - - if (resp->piggyback_content_length != 0) { - b->mem.pos = nxt_unit_sptr_get(&resp->piggyback_content); - b->mem.free = b->mem.pos + resp->piggyback_content_length; - - } else { - b->mem.pos = b->mem.free; - } - if (nxt_buf_mem_used_size(&b->mem) == 0) { next = b->next; b->next = NULL; diff --git a/src/nxt_router.h b/src/nxt_router.h index b7923dffa..56b7f7014 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -383,6 +383,10 @@ void nxt_router_test_app_rpc_create(nxt_task_t *task, /* The request deadline handler, for the app-timeout test. */ void nxt_router_test_app_timeout(nxt_task_t *task, void *obj, void *data); + +/* The application response decoder, for the response parse test. */ +nxt_int_t nxt_router_test_response_header_parse(nxt_task_t *task, + nxt_http_request_t *r, nxt_buf_t *b); #endif diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 26f18ae07..84115560e 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -231,6 +231,7 @@ static char * nxt_unit_snprint_prefix(char *p, char *end, pid_t pid, static void *nxt_unit_lvlhsh_alloc(void *data, size_t size); static void nxt_unit_lvlhsh_free(void *data, void *p); static int nxt_unit_memcasecmp(const void *p1, const void *p2, size_t length); +nxt_inline void nxt_unit_shm_copy(void *dst, const void *src, size_t size); /* @@ -276,6 +277,26 @@ nxt_unit_response_buf_size(uint32_t max_fields_count, */ +/* + * Copies "size" bytes out of shared memory, each read once through volatile. + * For a record whose size is not known before its first bytes are read. + */ + +nxt_inline void +nxt_unit_shm_copy(void *dst, const void *src, size_t size) +{ + u_char *d; + const volatile u_char *s; + + d = dst; + s = src; + + while (size-- != 0) { + *d++ = *s++; + } +} + + struct nxt_unit_mmap_buf_s { nxt_unit_buf_t buf; @@ -1469,9 +1490,10 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq) { int res; + char *method, *target, *preread; nxt_unit_impl_t *lib; nxt_unit_port_id_t port_id; - nxt_unit_request_t *r; + nxt_unit_request_t hdr; nxt_unit_mmap_buf_t *b; nxt_unit_request_info_t *req; nxt_unit_request_info_impl_t *req_impl; @@ -1492,18 +1514,23 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, } /* - * Validate every sptr in the request struct before any code path - * dereferences it. Offsets originate from the router (a more - * privileged peer) but the libunit ABI is also reachable from - * attacker-influenced input shapes; keeping the validation - * co-located with arrival makes the trust boundary explicit. + * The request is in a segment that every process of the application + * maps writable: the router wrote it, but a sibling process can change + * it at any time. So every value is read once. The fixed part and + * each field are copied out through volatile, the checks and the uses + * take the copies, and every sptr is resolved with + * nxt_unit_sptr_in_buf(), which reads the offset once. The pointers it + * returns are the ones used. */ { void *name, *value, *end; uint32_t i; + nxt_unit_field_t uf; nxt_unit_request_t *vr = recv_msg->start; uint32_t vsize = recv_msg->size; + hdr = *(volatile nxt_unit_request_t *) vr; + /* * The fields[] array trails the fixed request struct; its region * (fields_count * sizeof(nxt_unit_field_t)) must lie within the @@ -1513,36 +1540,39 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, * multiplication from overflowing a 32-bit fields_count. */ if (nxt_slow_path(sizeof(nxt_unit_request_t) - + (uint64_t) vr->fields_count + + (uint64_t) hdr.fields_count * sizeof(nxt_unit_field_t) > vsize)) { nxt_unit_warn(ctx, "#%"PRIu32": malformed request: fields_count " "%"PRIu32" exceeds buffer", recv_msg->stream, - vr->fields_count); + hdr.fields_count); return NXT_UNIT_ERROR; } + method = nxt_unit_sptr_in_buf(&vr->method, hdr.method_length, + recv_msg->start, vsize); + target = nxt_unit_sptr_in_buf(&vr->target, hdr.target_length, + recv_msg->start, vsize); + preread = nxt_unit_sptr_in_buf(&vr->preread_content, 0, + recv_msg->start, vsize); + if (nxt_slow_path( - !nxt_unit_sptr_in_buf(&vr->method, vr->method_length, - recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->version, vr->version_length, - recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->remote, vr->remote_length, + method == NULL || target == NULL || preread == NULL + || !nxt_unit_sptr_in_buf(&vr->version, hdr.version_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->local_addr, vr->local_addr_length, + || !nxt_unit_sptr_in_buf(&vr->remote, hdr.remote_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->local_port, vr->local_port_length, + || !nxt_unit_sptr_in_buf(&vr->local_addr, hdr.local_addr_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->server_name, vr->server_name_length, + || !nxt_unit_sptr_in_buf(&vr->local_port, hdr.local_port_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->target, vr->target_length, + || !nxt_unit_sptr_in_buf(&vr->server_name, + hdr.server_name_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->path, vr->path_length, + || !nxt_unit_sptr_in_buf(&vr->path, hdr.path_length, recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->query, vr->query_length, - recv_msg->start, vsize) - || !nxt_unit_sptr_in_buf(&vr->preread_content, 0, + || !nxt_unit_sptr_in_buf(&vr->query, hdr.query_length, recv_msg->start, vsize))) { nxt_unit_warn(ctx, "#%"PRIu32": malformed request: " @@ -1554,18 +1584,17 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, * Field strings must also lie past fields[]: the router puts them * there, and nxt_unit_request_group_dup_fields() moves fields one * slot on by subtracting sizeof(nxt_unit_field_t) from their - * offsets, which a target inside fields[] would underflow. The - * pointers the check returns are used, not the sptr read again: - * the peer can change the offset between two reads. + * offsets, which a target inside fields[] would underflow. */ - end = &vr->fields[vr->fields_count]; + end = &vr->fields[hdr.fields_count]; + + for (i = 0; i < hdr.fields_count; i++) { + uf = *(volatile nxt_unit_field_t *) &vr->fields[i]; - for (i = 0; i < vr->fields_count; i++) { - name = nxt_unit_sptr_in_buf(&vr->fields[i].name, - vr->fields[i].name_length, + name = nxt_unit_sptr_in_buf(&vr->fields[i].name, uf.name_length, recv_msg->start, vsize); value = nxt_unit_sptr_in_buf(&vr->fields[i].value, - vr->fields[i].value_length, + uf.value_length, recv_msg->start, vsize); if (nxt_slow_path(name == NULL || value == NULL @@ -1586,14 +1615,14 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, * read. Reject any that is neither "unset" nor a valid index. */ if (nxt_slow_path( - (vr->content_length_field != NXT_UNIT_NONE_FIELD - && vr->content_length_field >= vr->fields_count) - || (vr->content_type_field != NXT_UNIT_NONE_FIELD - && vr->content_type_field >= vr->fields_count) - || (vr->cookie_field != NXT_UNIT_NONE_FIELD - && vr->cookie_field >= vr->fields_count) - || (vr->authorization_field != NXT_UNIT_NONE_FIELD - && vr->authorization_field >= vr->fields_count))) + (hdr.content_length_field != NXT_UNIT_NONE_FIELD + && hdr.content_length_field >= hdr.fields_count) + || (hdr.content_type_field != NXT_UNIT_NONE_FIELD + && hdr.content_type_field >= hdr.fields_count) + || (hdr.cookie_field != NXT_UNIT_NONE_FIELD + && hdr.cookie_field >= hdr.fields_count) + || (hdr.authorization_field != NXT_UNIT_NONE_FIELD + && hdr.authorization_field >= hdr.fields_count))) { nxt_unit_warn(ctx, "#%"PRIu32": malformed request: cached field " "index out of range", recv_msg->stream); @@ -1619,12 +1648,11 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, req->response = NULL; req->response_buf = NULL; - r = req->request; - - req->content_length = r->content_length; + req->content_length = hdr.content_length; + /* The pointer from the check, not the sptr read again. */ req->content_buf = req->request_buf; - req->content_buf->free = nxt_unit_sptr_get(&r->preread_content); + req->content_buf->free = preread; req_impl->stream = recv_msg->stream; @@ -1648,11 +1676,9 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, req_impl->in_hash = 0; nxt_unit_debug(ctx, "#%"PRIu32": %.*s %.*s (%d)", recv_msg->stream, - (int) r->method_length, - (char *) nxt_unit_sptr_get(&r->method), - (int) r->target_length, - (char *) nxt_unit_sptr_get(&r->target), - (int) r->content_length); + (int) hdr.method_length, method, + (int) hdr.target_length, target, + (int) hdr.content_length); nxt_unit_port_id_init(&port_id, recv_msg->pid, recv_msg->reply_port); @@ -1918,8 +1944,9 @@ nxt_unit_send_req_headers_ack(nxt_unit_request_info_t *req) static int nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) { - size_t hsize; + size_t size, hsize; nxt_unit_impl_t *lib; + nxt_websocket_header_t wsh; nxt_unit_mmap_buf_t *b; nxt_unit_callbacks_t *cb; nxt_unit_request_info_t *req; @@ -1982,14 +2009,24 @@ nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) ws_impl->ws.header = (void *) b->buf.start; - if (nxt_slow_path((size_t) (b->buf.end - b->buf.start) < 2)) { + size = b->buf.end - b->buf.start; + + if (nxt_slow_path(size < 2)) { nxt_unit_warn(ctx, "#%"PRIu32": truncated websocket frame header", req_impl->stream); nxt_unit_websocket_frame_release(&ws_impl->ws); return NXT_UNIT_ERROR; } - hsize = nxt_websocket_frame_header_size(ws_impl->ws.header); + /* + * The frame is in shared memory a sibling process can write, so the + * header is copied out once: the two fixed bytes first, which give + * the header size, then the rest. The length and the mask flag + * come from the copy. + */ + nxt_unit_shm_copy(&wsh, b->buf.start, 2); + + hsize = nxt_websocket_frame_header_size(&wsh); /* * Reject truncated frames before reading the extended length / @@ -1998,21 +2035,24 @@ nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) * otherwise OOB-read b->buf.start + hsize - 4 (mask) and the * 8-byte extended length, and break the buffer invariant. */ - if (nxt_slow_path((size_t) (b->buf.end - b->buf.start) < hsize)) { + if (nxt_slow_path(size < hsize)) { nxt_unit_warn(ctx, "#%"PRIu32": truncated websocket frame: " "hsize %zu > buf size %zu", - req_impl->stream, hsize, - (size_t) (b->buf.end - b->buf.start)); + req_impl->stream, hsize, size); nxt_unit_websocket_frame_release(&ws_impl->ws); return NXT_UNIT_ERROR; } - ws_impl->ws.payload_len = nxt_websocket_frame_payload_len( - ws_impl->ws.header); + if (hsize > 2) { + nxt_unit_shm_copy(wsh.payload_len_, b->buf.start + 2, + nxt_min(hsize, sizeof(wsh)) - 2); + } + + ws_impl->ws.payload_len = nxt_websocket_frame_payload_len(&wsh); - if (ws_impl->ws.header->mask) { + if (wsh.mask) { ws_impl->ws.mask = (uint8_t *) b->buf.start + hsize - 4; } else { @@ -2267,9 +2307,10 @@ nxt_unit_field_hash(const char *name, size_t name_length) void nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) { - char *name; - uint32_t i, j; - nxt_unit_field_t *fields, f; + char *name, *jname; + void *start; + uint32_t i, j, n, size; + nxt_unit_field_t *fields, f, fi, fj; nxt_unit_request_t *r; static const nxt_str_t content_length = nxt_string("content-length"); @@ -2281,12 +2322,41 @@ nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) r = req->request; fields = r->fields; - for (i = 0; i < r->fields_count; i++) { - name = nxt_unit_sptr_get(&fields[i].name); + /* + * The request was checked on arrival, but it is in shared memory a + * sibling process can write, and this runs later. So the count is read + * once and checked again, each field is copied out before use, and the + * names are resolved with nxt_unit_sptr_in_buf(). A request that does + * not pass is left as it is. + */ + start = req->request_buf->start; + size = req->request_buf->end - req->request_buf->start; + + n = *(volatile uint32_t *) &r->fields_count; + + if (nxt_slow_path(sizeof(nxt_unit_request_t) + + (uint64_t) n * sizeof(nxt_unit_field_t) + > size)) + { + nxt_unit_req_warn(req, "group_dup_fields: fields_count %"PRIu32 + " exceeds buffer", n); + return; + } + + for (i = 0; i < n; i++) { + fi = *(volatile nxt_unit_field_t *) &fields[i]; + + name = nxt_unit_sptr_in_buf(&fields[i].name, fi.name_length, + start, size); + if (nxt_slow_path(name == NULL)) { + nxt_unit_req_warn(req, "group_dup_fields: field %"PRIu32 + " name out of buffer", i); + return; + } - switch (fields[i].hash) { + switch (fi.hash) { case NXT_UNIT_HASH_CONTENT_LENGTH: - if (fields[i].name_length == content_length.length + if (fi.name_length == content_length.length && nxt_unit_memcasecmp(name, content_length.start, content_length.length) == 0) { @@ -2296,7 +2366,7 @@ nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) break; case NXT_UNIT_HASH_CONTENT_TYPE: - if (fields[i].name_length == content_type.length + if (fi.name_length == content_type.length && nxt_unit_memcasecmp(name, content_type.start, content_type.length) == 0) { @@ -2306,7 +2376,7 @@ nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) break; case NXT_UNIT_HASH_COOKIE: - if (fields[i].name_length == cookie.length + if (fi.name_length == cookie.length && nxt_unit_memcasecmp(name, cookie.start, cookie.length) == 0) { @@ -2316,17 +2386,26 @@ nxt_unit_request_group_dup_fields(nxt_unit_request_info_t *req) break; } - for (j = i + 1; j < r->fields_count; j++) { - if (fields[i].hash != fields[j].hash - || fields[i].name_length != fields[j].name_length - || nxt_unit_memcasecmp(name, - nxt_unit_sptr_get(&fields[j].name), - fields[j].name_length) != 0) - { + for (j = i + 1; j < n; j++) { + fj = *(volatile nxt_unit_field_t *) &fields[j]; + + if (fi.hash != fj.hash || fi.name_length != fj.name_length) { + continue; + } + + jname = nxt_unit_sptr_in_buf(&fields[j].name, fj.name_length, + start, size); + if (nxt_slow_path(jname == NULL)) { + nxt_unit_req_warn(req, "group_dup_fields: field %"PRIu32 + " name out of buffer", j); + return; + } + + if (nxt_unit_memcasecmp(name, jname, fj.name_length) != 0) { continue; } - f = fields[j]; + f = fj; f.value.offset += (j - (i + 1)) * sizeof(f); while (j > i + 1) { diff --git a/src/test/nxt_router_response_parse_test.c b/src/test/nxt_router_response_parse_test.c new file mode 100644 index 000000000..73e3eb2b6 --- /dev/null +++ b/src/test/nxt_router_response_parse_test.c @@ -0,0 +1,197 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * nxt_router_response_header_parse() (src/nxt_router.c) decodes a response + * buffer the application wrote. Every count, length and sptr in it must be + * checked against the buffer, not followed. + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_RRP_NAME "X-Test" +#define NXT_RRP_VALUE "value" +#define NXT_RRP_BODY "body" +#define NXT_RRP_SIZE \ + (sizeof(nxt_unit_response_t) + sizeof(nxt_unit_field_t) \ + + nxt_length(NXT_RRP_NAME) + nxt_length(NXT_RRP_VALUE) \ + + nxt_length(NXT_RRP_BODY)) + + +/* A well-formed response with one field and a piggybacked body. */ +static void +nxt_router_response_parse_test_fill(nxt_unit_response_t *resp) +{ + u_char *p; + nxt_unit_field_t *f; + + nxt_memzero(resp, NXT_RRP_SIZE); + + resp->status = 200; + resp->fields_count = 1; + resp->piggyback_content_length = nxt_length(NXT_RRP_BODY); + + f = &resp->fields[0]; + f->name_length = nxt_length(NXT_RRP_NAME); + f->value_length = nxt_length(NXT_RRP_VALUE); + + p = (u_char *) &resp->fields[1]; + + nxt_unit_sptr_set(&f->name, p); + p = nxt_cpymem(p, NXT_RRP_NAME, nxt_length(NXT_RRP_NAME)); + + nxt_unit_sptr_set(&f->value, p); + p = nxt_cpymem(p, NXT_RRP_VALUE, nxt_length(NXT_RRP_VALUE)); + + nxt_unit_sptr_set(&resp->piggyback_content, p); + nxt_memcpy(p, NXT_RRP_BODY, nxt_length(NXT_RRP_BODY)); +} + + +static void +nxt_rrp_edit_name(nxt_unit_response_t *resp) +{ + resp->fields[0].name.offset = 4096; +} + + +static void +nxt_rrp_edit_value(nxt_unit_response_t *resp) +{ + resp->fields[0].value.offset = 4096; +} + + +static void +nxt_rrp_edit_value_length(nxt_unit_response_t *resp) +{ + resp->fields[0].value_length = 4096; +} + + +static void +nxt_rrp_edit_count(nxt_unit_response_t *resp) +{ + resp->fields_count = 2; +} + + +static void +nxt_rrp_edit_piggyback(nxt_unit_response_t *resp) +{ + resp->piggyback_content.offset = 4096; +} + + +static void +nxt_rrp_edit_piggyback_length(nxt_unit_response_t *resp) +{ + resp->piggyback_content_length = 4096; +} + + +nxt_int_t +nxt_router_response_parse_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_int_t ret; + nxt_uint_t i; + nxt_buf_t b; + nxt_task_t *task; + nxt_http_request_t *r; + + /* "resp" is in a page of its own, so a wrong offset stays mapped. */ + static union { + nxt_unit_response_t resp; + u_char page[8192]; + } u; + + static const struct { + const char *name; + void (*edit)(nxt_unit_response_t *resp); + size_t size; + nxt_int_t expect; + } tests[] = { + { "in-bounds response", NULL, NXT_RRP_SIZE, NXT_OK }, + { "short buffer", NULL, sizeof(nxt_unit_response_t) - 1, NXT_ERROR }, + { "fields_count past the buffer", nxt_rrp_edit_count, NXT_RRP_SIZE, + NXT_ERROR }, + { "name sptr out of the buffer", nxt_rrp_edit_name, NXT_RRP_SIZE, + NXT_ERROR }, + { "value sptr out of the buffer", nxt_rrp_edit_value, NXT_RRP_SIZE, + NXT_ERROR }, + { "value length past the buffer", nxt_rrp_edit_value_length, + NXT_RRP_SIZE, NXT_ERROR }, + { "piggyback sptr out of the buffer", nxt_rrp_edit_piggyback, + NXT_RRP_SIZE, NXT_ERROR }, + { "piggyback length past the buffer", nxt_rrp_edit_piggyback_length, + NXT_RRP_SIZE, NXT_ERROR }, + }; + + task = thr->task; + task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + ret = NXT_OK; + + for (i = 0; ret == NXT_OK && i < nxt_nitems(tests); i++) { + r = nxt_mp_zalloc(mp, sizeof(nxt_http_request_t)); + if (r == NULL) { + ret = NXT_ERROR; + break; + } + + r->mem_pool = mp; + r->task = *task; + + nxt_router_response_parse_test_fill(&u.resp); + + if (tests[i].edit != NULL) { + tests[i].edit(&u.resp); + } + + nxt_memzero(&b, sizeof(nxt_buf_t)); + b.mem.start = u.page; + b.mem.pos = u.page; + b.mem.free = u.page + tests[i].size; + b.mem.end = b.mem.free; + + if (nxt_router_test_response_header_parse(task, r, &b) + != tests[i].expect) + { + nxt_log_alert(task->log, "response parse test \"%s\" failed", + tests[i].name); + ret = NXT_ERROR; + break; + } + + if (tests[i].expect == NXT_OK + && (r->status != 200 + || nxt_buf_mem_used_size(&b.mem) != nxt_length(NXT_RRP_BODY) + || memcmp(b.mem.pos, NXT_RRP_BODY, nxt_length(NXT_RRP_BODY)) + != 0)) + { + nxt_log_alert(task->log, "response parse test \"%s\": " + "wrong status or body", tests[i].name); + ret = NXT_ERROR; + } + } + + nxt_mp_destroy(mp); + + if (ret == NXT_OK) { + nxt_log(task, NXT_LOG_NOTICE, "router response parse test passed"); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 7fdbcc244..d9415d073 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -72,6 +72,7 @@ nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), static nxt_int_t (*const nxt_security_tests[])(nxt_thread_t *) = { nxt_checked_test, nxt_port_mmap_read_test, + nxt_router_response_parse_test, }; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index c05a9691f..13fe7f96d 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -100,6 +100,7 @@ nxt_int_t nxt_port_rpc_fd_test(nxt_thread_t *thr); nxt_int_t nxt_port_queued_fd_test(nxt_thread_t *thr); nxt_int_t nxt_checked_test(nxt_thread_t *thr); nxt_int_t nxt_conn_close_test(nxt_thread_t *thr); +nxt_int_t nxt_router_response_parse_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); diff --git a/src/test/nxt_unit_msg_test.c b/src/test/nxt_unit_msg_test.c index 275bb57f5..83ea7706e 100644 --- a/src/test/nxt_unit_msg_test.c +++ b/src/test/nxt_unit_msg_test.c @@ -9,6 +9,11 @@ * partial last record past the message, mmap_id 0xFFFFFFFF wrapped the * lib->incoming index, and a segment of any size was accepted. Cases that * could crash the old code run in a child; a signal is a failure. + * + * The request cases put an nxt_unit_request_t into segment 0 the way the + * router does, and keep the test's own mapping of the segment: the segment + * is shared by every process of an application, so the test also plays a + * sibling process that writes the request after libunit checked it. */ #include "nxt_main.h" @@ -16,7 +21,10 @@ #include "nxt_port_queue.h" #include "nxt_app_queue.h" #include "nxt_unit.h" +#include "nxt_unit_request.h" +#include "nxt_unit_websocket.h" +#include #include #include #include @@ -26,11 +34,33 @@ #include -static int nxt_unit_msg_test_failures; -static int nxt_unit_msg_test_send_fails; -static int nxt_unit_msg_test_quit_called; -static nxt_unit_ctx_t *nxt_unit_msg_test_ctx; -static nxt_unit_ctx_t *nxt_unit_msg_test_follower; +/* What the request handler does with a request that arrived. */ +typedef enum { + NXT_UNIT_MSG_TEST_READ_BODY = 0, + NXT_UNIT_MSG_TEST_GROUP_DUP, + NXT_UNIT_MSG_TEST_RAISE_COUNT, + NXT_UNIT_MSG_TEST_UPGRADE, +} nxt_unit_msg_test_mode_t; + + +#define NXT_UNIT_MSG_TEST_ROUTER_PORT 2 +#define NXT_UNIT_MSG_TEST_STREAM 7 +#define NXT_UNIT_MSG_TEST_BODY "ab" + + +static int nxt_unit_msg_test_failures; +static int nxt_unit_msg_test_send_fails; +static int nxt_unit_msg_test_quit_called; +static int nxt_unit_msg_test_handler_calls; +static int nxt_unit_msg_test_handler_ok; +static int nxt_unit_msg_test_ws_calls; +static int nxt_unit_msg_test_ws_ok; +static int nxt_unit_msg_test_close_calls; +static nxt_unit_msg_test_mode_t nxt_unit_msg_test_mode; +static pid_t nxt_unit_msg_test_pid; +static nxt_unit_ctx_t *nxt_unit_msg_test_ctx; +static nxt_unit_ctx_t *nxt_unit_msg_test_follower; +static nxt_port_mmap_header_t *nxt_unit_msg_test_seg0; static int nxt_unit_msg_test_send_records_to(nxt_unit_ctx_t *ctx, const nxt_port_mmap_msg_t *records, size_t nrecords, size_t tail); @@ -53,6 +83,93 @@ nxt_unit_msg_test_assert(int cond, const char *name) static void nxt_unit_msg_test_handler(nxt_unit_request_info_t *req) { + char buf[16]; + ssize_t n; + nxt_unit_field_t *f; + nxt_unit_request_t *r; + + nxt_unit_msg_test_handler_calls++; + + r = req->request; + + switch (nxt_unit_msg_test_mode) { + + case NXT_UNIT_MSG_TEST_READ_BODY: + n = nxt_unit_request_read(req, buf, sizeof(buf)); + + nxt_unit_msg_test_handler_ok = + n == (ssize_t) nxt_length(NXT_UNIT_MSG_TEST_BODY) + && memcmp(buf, NXT_UNIT_MSG_TEST_BODY, n) == 0; + + break; + + case NXT_UNIT_MSG_TEST_GROUP_DUP: + nxt_unit_request_group_dup_fields(req); + + /* "X-Dup" and "x-dup" are now neighbours, with one name. */ + f = r->fields; + + nxt_unit_msg_test_handler_ok = + r->fields_count == 3 + && f[1].name_length == nxt_length("X-Dup") + && nxt_unit_sptr_get(&f[1].name) == nxt_unit_sptr_get(&f[2].name) + && memcmp(nxt_unit_sptr_get(&f[1].value), "1", 1) == 0 + && memcmp(nxt_unit_sptr_get(&f[2].value), "2", 1) == 0 + && f[0].name_length == nxt_length("Host") + && memcmp(nxt_unit_sptr_get(&f[0].value), "localhost", 9) == 0; + + break; + + case NXT_UNIT_MSG_TEST_RAISE_COUNT: + /* + * A sibling process raises the count after libunit checked it. + * The request sits in the last chunk of the segment, so a walk + * over that many fields leaves the mapping. + */ + r->fields_count = 0x00FFFFFF; + + nxt_unit_request_group_dup_fields(req); + + nxt_unit_msg_test_handler_ok = 1; + + break; + + case NXT_UNIT_MSG_TEST_UPGRADE: + nxt_unit_msg_test_handler_ok = + nxt_unit_response_init(req, 101, 0, 0) == NXT_UNIT_OK + && nxt_unit_response_upgrade(req) == NXT_UNIT_OK + && nxt_unit_response_send(req) == NXT_UNIT_OK; + + /* The request now waits for frames; the close finishes it. */ + return; + } + + nxt_unit_request_done(req, NXT_UNIT_ERROR); +} + + +static void +nxt_unit_msg_test_websocket(nxt_unit_websocket_frame_t *ws) +{ + char buf[16]; + ssize_t n; + + nxt_unit_msg_test_ws_calls++; + + n = nxt_unit_websocket_read(ws, buf, sizeof(buf)); + + nxt_unit_msg_test_ws_ok = ws->payload_len == 3 && n == 3 + && memcmp(buf, "abc", 3) == 0; + + nxt_unit_websocket_done(ws); +} + + +static void +nxt_unit_msg_test_close(nxt_unit_request_info_t *req) +{ + nxt_unit_msg_test_close_calls++; + nxt_unit_request_done(req, NXT_UNIT_ERROR); } @@ -152,7 +269,13 @@ nxt_unit_msg_test_send_segment(size_t size, uint32_t id) hdr->src_pid = getpid(); hdr->dst_pid = getpid(); - munmap(hdr, size); + /* The test keeps segment 0 mapped: the request cases write into it. */ + if (id == 0 && nxt_unit_msg_test_seg0 == NULL && size == PORT_MMAP_SIZE) { + nxt_unit_msg_test_seg0 = hdr; + + } else { + munmap(hdr, size); + } memset(&msg, 0, sizeof(msg)); @@ -436,6 +559,417 @@ nxt_unit_msg_test_dup_id_case(void *data) } +/* + * A request as the router lays it out (nxt_router_prepare_msg()): the + * struct, fields[], the strings, then the preread body. Written at "p", + * with every sptr set; the size is returned. + */ +static uint32_t +nxt_unit_msg_test_write_request(u_char *p) +{ + u_char *s; + nxt_unit_field_t *f; + nxt_unit_request_t *r; + + static const struct { + const char *name; + const char *value; + } fields[] = { + { "Host", "localhost" }, + { "X-Dup", "1" }, + { "x-dup", "2" }, + }; + + r = (nxt_unit_request_t *) p; + + memset(r, 0, sizeof(nxt_unit_request_t)); + + r->fields_count = nxt_nitems(fields); + + r->content_length_field = NXT_UNIT_NONE_FIELD; + r->content_type_field = NXT_UNIT_NONE_FIELD; + r->cookie_field = NXT_UNIT_NONE_FIELD; + r->authorization_field = NXT_UNIT_NONE_FIELD; + + r->content_length = nxt_length(NXT_UNIT_MSG_TEST_BODY); + + s = (u_char *) &r->fields[r->fields_count]; + +#define NXT_UNIT_MSG_TEST_STR(sptr, len, str) \ + do { \ + nxt_unit_sptr_set(&r->sptr, s); \ + r->len = nxt_length(str); \ + s = nxt_cpymem(s, str, nxt_length(str)); \ + *s++ = '\0'; \ + } while (0) + + NXT_UNIT_MSG_TEST_STR(method, method_length, "GET"); + NXT_UNIT_MSG_TEST_STR(version, version_length, "HTTP/1.1"); + NXT_UNIT_MSG_TEST_STR(remote, remote_length, "127.0.0.1"); + NXT_UNIT_MSG_TEST_STR(local_addr, local_addr_length, "127.0.0.1"); + NXT_UNIT_MSG_TEST_STR(local_port, local_port_length, "8080"); + NXT_UNIT_MSG_TEST_STR(server_name, server_name_length, "localhost"); + NXT_UNIT_MSG_TEST_STR(target, target_length, "/a?b=c"); + NXT_UNIT_MSG_TEST_STR(path, path_length, "/a"); + NXT_UNIT_MSG_TEST_STR(query, query_length, "b=c"); + +#undef NXT_UNIT_MSG_TEST_STR + + for (f = r->fields; f < &r->fields[r->fields_count]; f++) { + f->hash = nxt_unit_field_hash(fields[f - r->fields].name, + strlen(fields[f - r->fields].name)); + f->skip = 0; + f->hopbyhop = 0; + + nxt_unit_sptr_set(&f->name, s); + f->name_length = strlen(fields[f - r->fields].name); + s = nxt_cpymem(s, fields[f - r->fields].name, f->name_length); + *s++ = '\0'; + + nxt_unit_sptr_set(&f->value, s); + f->value_length = strlen(fields[f - r->fields].value); + s = nxt_cpymem(s, fields[f - r->fields].value, f->value_length); + *s++ = '\0'; + } + + nxt_unit_sptr_set(&r->preread_content, s); + s = nxt_cpymem(s, NXT_UNIT_MSG_TEST_BODY, + nxt_length(NXT_UNIT_MSG_TEST_BODY)); + + return s - p; +} + + +/* One message about the request, with one record over "chunk". */ +static int +nxt_unit_msg_test_send_chunk(uint8_t type, uint8_t last, nxt_chunk_id_t chunk, + uint32_t size) +{ + u_char buf[sizeof(nxt_port_msg_t) + + sizeof(nxt_port_mmap_msg_t)]; + nxt_port_msg_t msg; + nxt_port_mmap_msg_t rec; + + memset(&msg, 0, sizeof(msg)); + + /* The router port was registered with the pid libunit started with. */ + msg.stream = NXT_UNIT_MSG_TEST_STREAM; + msg.pid = nxt_unit_msg_test_pid; + msg.reply_port = NXT_UNIT_MSG_TEST_ROUTER_PORT; + msg.type = type; + msg.last = last; + msg.mmap = 1; + + rec.mmap_id = 0; + rec.chunk_id = chunk; + rec.size = size; + + memcpy(buf, &msg, sizeof(msg)); + memcpy(buf + sizeof(msg), &rec, sizeof(rec)); + + return nxt_unit_test_process_msg(nxt_unit_msg_test_ctx, buf, sizeof(buf), + -1); +} + + +/* Writes a request into "chunk", lets "edit" damage it, and sends it. */ +static int +nxt_unit_msg_test_send_request(nxt_chunk_id_t chunk, + void (*edit)(nxt_unit_request_t *r)) +{ + u_char *p; + uint32_t size; + + p = nxt_port_mmap_chunk_start(nxt_unit_msg_test_seg0, chunk); + + size = nxt_unit_msg_test_write_request(p); + + if (edit != NULL) { + edit((nxt_unit_request_t *) p); + } + + nxt_unit_msg_test_handler_calls = 0; + nxt_unit_msg_test_handler_ok = 0; + + return nxt_unit_msg_test_send_chunk(_NXT_PORT_MSG_REQ_HEADERS, 1, chunk, + size); +} + + +static void +nxt_unit_msg_test_edit_target(nxt_unit_request_t *r) +{ + r->target.offset = 0x7FFFFFF0; +} + + +static void +nxt_unit_msg_test_edit_preread(nxt_unit_request_t *r) +{ + r->preread_content.offset = 0x7FFFFFF0; +} + + +static void +nxt_unit_msg_test_edit_count(nxt_unit_request_t *r) +{ + r->fields_count = 0x10000000; +} + + +static void +nxt_unit_msg_test_edit_index(nxt_unit_request_t *r) +{ + r->cookie_field = r->fields_count; +} + + +static void +nxt_unit_msg_test_edit_field_name(nxt_unit_request_t *r) +{ + r->fields[1].name.offset = 0x7FFFFFF0; +} + + +static void +nxt_unit_msg_test_edit_field_inside(nxt_unit_request_t *r) +{ + nxt_unit_sptr_set(&r->fields[1].name, &r->fields[0]); +} + + +static void +nxt_unit_msg_test_edit_handshake(nxt_unit_request_t *r) +{ + r->websocket_handshake = 1; +} + + +typedef struct { + const char *name; + void (*edit)(nxt_unit_request_t *r); + nxt_unit_msg_test_mode_t mode; + nxt_chunk_id_t chunk; + int expect; + int handler_calls; +} nxt_unit_msg_test_request_t; + + +static const nxt_unit_msg_test_request_t requests[] = { + { "well-formed request reaches the handler", NULL, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_OK, 1 }, + { "target sptr out of buffer is refused", nxt_unit_msg_test_edit_target, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "preread sptr out of buffer is refused", nxt_unit_msg_test_edit_preread, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "fields_count past the buffer is refused", nxt_unit_msg_test_edit_count, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "cached field index out of range is refused", + nxt_unit_msg_test_edit_index, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "field name sptr out of buffer is refused", + nxt_unit_msg_test_edit_field_name, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "field name inside fields[] is refused", + nxt_unit_msg_test_edit_field_inside, + NXT_UNIT_MSG_TEST_READ_BODY, 2, NXT_UNIT_ERROR, 0 }, + { "duplicate fields are grouped", NULL, + NXT_UNIT_MSG_TEST_GROUP_DUP, 2, NXT_UNIT_OK, 1 }, + { "fields_count raised after the check stays in the buffer", NULL, + NXT_UNIT_MSG_TEST_RAISE_COUNT, PORT_MMAP_CHUNK_COUNT - 1, NXT_UNIT_OK, + 1 }, +}; + + +static int +nxt_unit_msg_test_request_case(void *data) +{ + int rc; + const nxt_unit_msg_test_request_t *t = data; + + nxt_unit_msg_test_mode = t->mode; + + rc = nxt_unit_msg_test_send_request(t->chunk, t->edit); + + if (nxt_unit_msg_test_handler_calls != t->handler_calls) { + printf("unit msg test: handler called %d times, expected %d\n", + nxt_unit_msg_test_handler_calls, t->handler_calls); + return 1; + } + + if (t->handler_calls != 0 && !nxt_unit_msg_test_handler_ok) { + printf("unit msg test: handler saw a wrong request\n"); + return 2; + } + + return NXT_UNIT_MSG_TEST_RC(rc); +} + + +/* + * A sibling process that keeps changing the preread offset while the + * request is checked and then used. libunit resolved the sptr once for + * the check and again for the body start, so a change in between put the + * body start 2 GB past the buffer, and the handler's read went there. + */ + +typedef struct { + volatile uint32_t *offset; + uint32_t good; + volatile int stop; +} nxt_unit_msg_test_racer_t; + + +static void * +nxt_unit_msg_test_racer(void *data) +{ + nxt_unit_msg_test_racer_t *racer = data; + + while (!racer->stop) { + *racer->offset = 0x7FFFFFF0; + *racer->offset = racer->good; + } + + return NULL; +} + + +static int +nxt_unit_msg_test_preread_race_case(void *data) +{ + int i, rc, accepted; + u_char *p; + uint32_t size; + pthread_t thread; + nxt_unit_request_t *r; + nxt_unit_msg_test_racer_t racer; + + nxt_unit_msg_test_mode = NXT_UNIT_MSG_TEST_READ_BODY; + + p = nxt_port_mmap_chunk_start(nxt_unit_msg_test_seg0, 2); + r = (nxt_unit_request_t *) p; + + size = nxt_unit_msg_test_write_request(p); + + racer.offset = &r->preread_content.offset; + racer.good = r->preread_content.offset; + racer.stop = 0; + + if (pthread_create(&thread, NULL, nxt_unit_msg_test_racer, &racer) != 0) { + return 1; + } + + accepted = 0; + + for (i = 0; i < 20000; i++) { + /* Releasing the request wipes the chunk; write it again. */ + (void) nxt_unit_msg_test_write_request(p); + + nxt_unit_msg_test_handler_calls = 0; + nxt_unit_msg_test_handler_ok = 0; + + rc = nxt_unit_msg_test_send_chunk(_NXT_PORT_MSG_REQ_HEADERS, 1, 2, + size); + + if (rc == NXT_UNIT_OK) { + if (nxt_unit_msg_test_handler_calls != 1 + || !nxt_unit_msg_test_handler_ok) + { + racer.stop = 1; + pthread_join(thread, NULL); + + printf("unit msg test: accepted request, wrong body\n"); + return 2; + } + + accepted++; + } + } + + racer.stop = 1; + pthread_join(thread, NULL); + + if (accepted == 0) { + printf("unit msg test: no request was accepted under the race\n"); + return 3; + } + + return NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK); +} + + +/* A request upgraded to a websocket, then one frame in shared memory. */ +static int +nxt_unit_msg_test_websocket_case(void *data) +{ + int rc, whole; + u_char *p; + + static const u_char mask[4] = { 0x10, 0x20, 0x30, 0x40 }; + + whole = *(const int *) data; + + nxt_unit_msg_test_mode = NXT_UNIT_MSG_TEST_UPGRADE; + + rc = nxt_unit_msg_test_send_request(2, nxt_unit_msg_test_edit_handshake); + + if (rc != NXT_UNIT_OK || nxt_unit_msg_test_handler_calls != 1 + || !nxt_unit_msg_test_handler_ok) + { + printf("unit msg test: upgrade failed: %d\n", rc); + return 1; + } + + /* A masked text frame with a 2-byte extended length: "abc". */ + p = nxt_port_mmap_chunk_start(nxt_unit_msg_test_seg0, 3); + + p[0] = 0x81; + p[1] = 0x80 | 126; + p[2] = 0; + p[3] = 3; + memcpy(p + 4, mask, 4); + p[8] = 'a' ^ mask[0]; + p[9] = 'b' ^ mask[1]; + p[10] = 'c' ^ mask[2]; + + nxt_unit_msg_test_ws_calls = 0; + nxt_unit_msg_test_ws_ok = 0; + + rc = nxt_unit_msg_test_send_chunk(_NXT_PORT_MSG_WEBSOCKET, 0, 3, + whole ? 11 : 2); + + if (whole) { + if (rc != NXT_UNIT_OK || nxt_unit_msg_test_ws_calls != 1 + || !nxt_unit_msg_test_ws_ok) + { + printf("unit msg test: frame: rc %d, handler %d ok %d\n", rc, + nxt_unit_msg_test_ws_calls, nxt_unit_msg_test_ws_ok); + return 2; + } + + } else { + /* Two bytes of a header that says it is eight. */ + if (rc != NXT_UNIT_ERROR || nxt_unit_msg_test_ws_calls != 0) { + printf("unit msg test: short frame: rc %d, handler %d\n", rc, + nxt_unit_msg_test_ws_calls); + return 3; + } + } + + nxt_unit_msg_test_close_calls = 0; + + rc = nxt_unit_msg_test_send_chunk(_NXT_PORT_MSG_WEBSOCKET, 1, 3, 0); + + if (rc != NXT_UNIT_OK || nxt_unit_msg_test_close_calls != 1) { + printf("unit msg test: close: rc %d, handler %d\n", rc, + nxt_unit_msg_test_close_calls); + return 4; + } + + return NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK); +} + + int main(void) { @@ -445,6 +979,8 @@ main(void) nxt_unit_init_t init; nxt_port_mmap_msg_t rec[2]; + static const int whole_frame = 1, short_frame = 0; + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, ready) == -1 || socketpair(AF_UNIX, SOCK_DGRAM, 0, router) == -1 || socketpair(AF_UNIX, SOCK_DGRAM, 0, read) == -1 @@ -454,9 +990,13 @@ main(void) return 1; } + nxt_unit_msg_test_pid = getpid(); + memset(&init, 0, sizeof(init)); init.callbacks.request_handler = nxt_unit_msg_test_handler; + init.callbacks.websocket_handler = nxt_unit_msg_test_websocket; + init.callbacks.close_handler = nxt_unit_msg_test_close; init.callbacks.port_send = nxt_unit_msg_test_send; init.callbacks.port_recv = nxt_unit_msg_test_recv; init.callbacks.quit = nxt_unit_msg_test_quit; @@ -469,7 +1009,7 @@ main(void) init.ready_stream = 1; init.router_port.id.pid = getpid(); - init.router_port.id.id = 2; + init.router_port.id.id = NXT_UNIT_MSG_TEST_ROUTER_PORT; init.router_port.in_fd = -1; init.router_port.out_fd = router[0]; @@ -556,6 +1096,25 @@ main(void) nxt_unit_msg_test_dup_id_case, NULL, NXT_UNIT_OK); + for (i = 0; i < nxt_nitems(requests); i++) { + nxt_unit_msg_test_in_child(requests[i].name, + nxt_unit_msg_test_request_case, + (void *) &requests[i], requests[i].expect); + } + + nxt_unit_msg_test_in_child("preread offset changed after the check is " + "not followed", + nxt_unit_msg_test_preread_race_case, NULL, + NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("masked websocket frame in shared memory", + nxt_unit_msg_test_websocket_case, + (void *) &whole_frame, NXT_UNIT_OK); + + nxt_unit_msg_test_in_child("truncated websocket frame is refused", + nxt_unit_msg_test_websocket_case, + (void *) &short_frame, NXT_UNIT_OK); + if (nxt_unit_msg_test_failures != 0) { printf("unit msg test: %d failure(s)\n", nxt_unit_msg_test_failures); return 1; From 9289c02d5d130f6b11d70b0608472bdc180daa88 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 00:04:23 +0000 Subject: [PATCH 298/391] port: bound the shared memory segment ids A segment id is an index into the array of segments, and the array is grown to hold the id. In libunit the incoming id comes from the peer: from an mmap record in a port message, or from a segment header in memory the peer can still write. nxt_unit_mmap_at() refused only UINT32_MAX, so an id like 100000000 in one message made the application allocate and initialise an array of that many slots: a CPU and memory denial of service. The router already refused such ids from applications: nxt_port_incoming_port_mmap() checks NXT_PORT_MMAP_MAX_SEGMENTS (about 410 in a normal build) before it grows the array. But nxt_port_mmap_at() itself had no limit, and nothing limited the segments the router creates. Add NXT_PORT_MMAPS_MAX, 65536 segments per process pair and direction. Each segment is about 10 MiB, so no real peer reaches it. Both growth functions refuse an id at or past the limit before any growth, and nxt_port_new_port_mmap() does not create a segment whose id would reach it. On the router this is defence in depth, and a real limit in NXT_MMAP_TINY_CHUNK builds. Every caller already handles the NULL as an error. Tests: the libunit rows in src/test/nxt_unit_msg_test.c, and a new src/test/nxt_port_mmaps_max_test.c for the router side, through two NXT_TESTS hooks in src/nxt_port_memory.c. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 8 ++ auto/sources | 1 + docs/changes.xml | 11 +++ src/nxt_port_memory.c | 42 ++++++++- src/nxt_port_memory.h | 7 ++ src/nxt_port_memory_int.h | 26 ++++++ src/nxt_unit.c | 22 ++--- src/test/nxt_port_mmaps_max_test.c | 138 +++++++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + src/test/nxt_unit_msg_test.c | 16 +++- 11 files changed, 260 insertions(+), 16 deletions(-) create mode 100644 src/test/nxt_port_mmaps_max_test.c diff --git a/CHANGES b/CHANGES index 0c2bff6ce..6855e8030 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: an application used a shared memory segment id from the + other process to grow its array of segments without a limit. An id + like 100000000 in one message made it allocate and fill an array of + that many slots, so the process could run out of memory. It now + refuses an id of 65536 or more. The router already refused such ids + from applications; it now uses the same limit for the segments it + creates. + *) Bugfix: the proxy recognised an upstream Transfer-Encoding only when its value was exactly "chunked". "Chunked" or "gzip, chunked" was not recognised, so the body was framed by Content-Length or by diff --git a/auto/sources b/auto/sources index ac5185d92..dca3c776f 100644 --- a/auto/sources +++ b/auto/sources @@ -186,6 +186,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_use_unless_zero_test.c \ src/test/nxt_port_mmap_range_test.c \ src/test/nxt_port_mmap_read_test.c \ + src/test/nxt_port_mmaps_max_test.c \ src/test/nxt_port_ready_test.c \ src/test/nxt_router_new_port_test.c \ src/test/nxt_router_start_fail_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 7fa6fc70f..6156d72fd 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,17 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +an application used a shared memory segment id from the other process to grow +its array of segments without a limit. An id like 100000000 in one message +made it allocate and fill an array of that many slots, so the process could +run out of memory. It now refuses an id of 65536 or more. The router already +refused such ids from applications; it now uses the same limit for the +segments it creates. + + + The proxy recognised an upstream Transfer-Encoding only when its value was diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 2cde6acb2..2fa843efd 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -59,10 +59,11 @@ nxt_port_mmap_at(nxt_port_mmaps_t *port_mmaps, uint32_t i) } /* - * A capacity able to hold slot i is i + 1 elements, which is not - * representable in uint32_t for i == UINT32_MAX. + * "i" is a segment id from the peer, or the next id of an outgoing + * segment. Past NXT_PORT_MMAPS_MAX it is refused before any growth, + * so a huge id costs nothing. This also keeps i + 1 in uint32_t. */ - if (nxt_slow_path(i == UINT32_MAX)) { + if (nxt_slow_path(i >= NXT_PORT_MMAPS_MAX)) { return NULL; } @@ -404,6 +405,17 @@ nxt_port_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, nxt_int_t n) nxt_port_mmap_header_t *hdr; nxt_port_mmap_handler_t *mmap_handler; + /* + * The id of the new segment is its index, mmaps->size. Refused here + * with its own message; nxt_port_mmap_at() below refuses it too. + */ + if (nxt_slow_path(mmaps->size >= NXT_PORT_MMAPS_MAX)) { + nxt_alert(task, "too many port mmaps (%uD), limit is %uD", + mmaps->size, NXT_PORT_MMAPS_MAX); + + return NULL; + } + mmap_handler = nxt_zalloc(sizeof(nxt_port_mmap_handler_t)); if (nxt_slow_path(mmap_handler == NULL)) { nxt_alert(task, "failed to allocate mmap_handler"); @@ -1127,3 +1139,27 @@ nxt_port_broadcast_shm_ack(nxt_task_t *task, nxt_port_t *port, void *data) nxt_process_use(task, process, -1); } + + +#if (NXT_TESTS) + +/* + * The static growth and creation paths, for + * src/test/nxt_port_mmaps_max_test.c. + */ + +nxt_port_mmap_t * +nxt_port_test_mmap_at(nxt_port_mmaps_t *mmaps, uint32_t i) +{ + return nxt_port_mmap_at(mmaps, i); +} + + +nxt_port_mmap_handler_t * +nxt_port_test_new_port_mmap(nxt_task_t *task, nxt_port_mmaps_t *mmaps, + nxt_int_t n) +{ + return nxt_port_new_port_mmap(task, mmaps, n); +} + +#endif diff --git a/src/nxt_port_memory.h b/src/nxt_port_memory.h index f1e709646..476af6117 100644 --- a/src/nxt_port_memory.h +++ b/src/nxt_port_memory.h @@ -52,4 +52,11 @@ nxt_int_t nxt_shm_open(nxt_task_t *task, size_t size); void nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process); +#if (NXT_TESTS) +/* The segment array growth and the segment creation, for the limit test. */ +nxt_port_mmap_t *nxt_port_test_mmap_at(nxt_port_mmaps_t *mmaps, uint32_t i); +nxt_port_mmap_handler_t *nxt_port_test_new_port_mmap(nxt_task_t *task, + nxt_port_mmaps_t *mmaps, nxt_int_t n); +#endif + #endif /* _NXT_PORT_MEMORY_H_INCLUDED_ */ diff --git a/src/nxt_port_memory_int.h b/src/nxt_port_memory_int.h index d7fd4e083..846cf1763 100644 --- a/src/nxt_port_memory_int.h +++ b/src/nxt_port_memory_int.h @@ -48,6 +48,32 @@ */ #define NXT_PORT_MMAP_MAX_SEGMENTS (UINT32_MAX / PORT_MMAP_DATA_SIZE + 1) +/* + * The most shared memory segments one process keeps for one peer, in each + * direction. A segment id is an index into that array, and the array is + * grown to hold the id. On both sides, the router and libunit, an incoming + * id is taken from the peer: from an mmap record in a port message, or from + * a segment header in memory the peer can still write. Without a limit an + * id like 100000000 makes the receiver allocate and initialise an array of + * that many slots from one message. + * + * NXT_PORT_MMAP_MAX_SEGMENTS above bounds only the libunit-authored ids the + * router receives. The router's own outgoing segments are not bounded by + * shm_limit, so libunit needs a limit of its own, and this one is used by + * both sides, in nxt_port_mmap_at() and nxt_unit_mmap_at(). Each segment + * holds PORT_MMAP_SIZE bytes, about 10 MiB, so 65536 segments are about + * 640 GiB of shared memory between one pair of processes: no real peer + * reaches it. The arrays a bad id can grow are bounded too: at the limit + * the router's array of nxt_port_mmap_t (one pointer) is 512 KiB, and + * libunit's array of nxt_unit_mmap_t (a pointer, a pthread_t and a queue, + * 32 bytes on 64-bit) is 2 MiB. The router refuses to create a segment + * past the limit, and both sides refuse an incoming id past it. + * + * In an NXT_MMAP_TINY_CHUNK build a segment is 2 KiB, so the limit is only + * 128 MiB of shared memory per pair; that build is for debugging. + */ +#define NXT_PORT_MMAPS_MAX 65536U + typedef uint32_t nxt_chunk_id_t; diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 84115560e..4b7793fb0 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -4556,13 +4556,14 @@ nxt_unit_mmap_at(nxt_unit_mmaps_t *mmaps, uint32_t i) /* * The same guards as nxt_port_mmap_at() on the router side. "i" is a * segment id taken from the peer (an mmap record or a segment header), - * and is deliberately not capped: the router allocates its outgoing - * segments without a limit. What has to hold is the arithmetic: a - * capacity able to hold slot i is i + 1 elements, which does not fit - * uint32_t for i == UINT32_MAX -- "i + 1 > cap" then wraps to 0, skips - * the growth, and the element pointer lands 4G elements past the array. + * or the next id of an outgoing segment. Past NXT_PORT_MMAPS_MAX it is + * refused before any growth: the array is grown to hold slot i, so an + * id like 100000000 would otherwise cost a huge allocation and its + * initialisation from one message. The limit also keeps i + 1 in + * uint32_t; for i == UINT32_MAX "i + 1 > cap" wraps to 0, skips the + * growth, and the element pointer lands 4G elements past the array. */ - if (nxt_slow_path(i == UINT32_MAX)) { + if (nxt_slow_path(i >= NXT_PORT_MMAPS_MAX)) { return NULL; } @@ -4934,8 +4935,8 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) /* * The segment id lives in memory the sender keeps mapped writable: read * it once and use only the copy. It indexes lib->incoming, which - * nxt_unit_mmap_at() grows to fit and which refuses an id whose slot - * cannot be represented. + * nxt_unit_mmap_at() grows to fit and which refuses an id past + * NXT_PORT_MMAPS_MAX. */ id = hdr->id; @@ -5178,9 +5179,8 @@ nxt_unit_check_rbuf_mmap(nxt_unit_ctx_t *ctx, nxt_unit_mmaps_t *mmaps, * anything is allocated, instead of its last "record" being read past the * end of the message. Each record is copied out before use, and every * field is bounds-checked before it reaches an index or pointer arithmetic: - * mmap_id by nxt_unit_mmap_at(), which refuses an id whose slot cannot be - * represented (the id is not capped otherwise -- see there), chunk_id and - * size by nxt_port_mmap_chunk_range_valid(). + * mmap_id by nxt_unit_mmap_at(), which refuses an id past NXT_PORT_MMAPS_MAX + * (see there), chunk_id and size by nxt_port_mmap_chunk_range_valid(). */ static int diff --git a/src/test/nxt_port_mmaps_max_test.c b/src/test/nxt_port_mmaps_max_test.c new file mode 100644 index 000000000..61c65d7f0 --- /dev/null +++ b/src/test/nxt_port_mmaps_max_test.c @@ -0,0 +1,138 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * NXT_PORT_MMAPS_MAX on the router side (src/nxt_port_memory.c). The + * array of segments is grown to hold the id it is asked for, and an + * incoming id is the peer's. nxt_port_mmap_at() used to grow the array for + * any id below UINT32_MAX, so one message with mmap_id 100000000 cost an + * allocation and initialisation of that many slots. Now the last id below + * the limit is accepted, the limit and everything past it are refused + * before any growth, and nxt_port_new_port_mmap() does not create a segment + * whose id would reach the limit. + * + * Cost: the boundary cases grow the array to NXT_PORT_MMAPS_MAX slots of + * one pointer, 512 KiB, and create one real segment of PORT_MMAP_SIZE + * (about 10 MiB) so that the last allowed id is really created. The + * segment is a sparse memfd or shm file and only its header page is + * touched, so the test takes well under a millisecond and no measurable + * RSS; it needs 10 MiB of /dev/shm on paper only. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +nxt_int_t +nxt_port_mmaps_max_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_port_mmap_t *port_mmap; + nxt_port_mmaps_t mmaps; + nxt_port_mmap_handler_t *mmap_handler; + + nxt_thread_time_update(thr); + + task = thr->task; + task->thread = thr; + + nxt_memzero(&mmaps, sizeof(mmaps)); + + if (nxt_slow_path(nxt_thread_mutex_create(&mmaps.mutex) != NXT_OK)) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + + /* A huge id is refused before the array is touched. */ + + port_mmap = nxt_port_test_mmap_at(&mmaps, 100000000); + + if (port_mmap != NULL || mmaps.elts != NULL || mmaps.cap != 0 + || mmaps.size != 0) + { + nxt_log_alert(thr->log, "port mmaps max test: id 100000000 was " + "accepted, or the array was grown for it"); + goto done; + } + + port_mmap = nxt_port_test_mmap_at(&mmaps, NXT_PORT_MMAPS_MAX); + + if (port_mmap != NULL || mmaps.elts != NULL || mmaps.cap != 0) { + nxt_log_alert(thr->log, "port mmaps max test: id %uD was accepted", + NXT_PORT_MMAPS_MAX); + goto done; + } + + port_mmap = nxt_port_test_mmap_at(&mmaps, UINT32_MAX); + + if (port_mmap != NULL || mmaps.elts != NULL || mmaps.cap != 0) { + nxt_log_alert(thr->log, "port mmaps max test: id UINT32_MAX was " + "accepted"); + goto done; + } + + /* An id just below the limit grows the array to MAX - 1 slots. */ + + port_mmap = nxt_port_test_mmap_at(&mmaps, NXT_PORT_MMAPS_MAX - 2); + + if (port_mmap == NULL || mmaps.size != NXT_PORT_MMAPS_MAX - 1 + || port_mmap != mmaps.elts + NXT_PORT_MMAPS_MAX - 2 + || port_mmap->mmap_handler != NULL) + { + nxt_log_alert(thr->log, "port mmaps max test: id %uD was refused", + NXT_PORT_MMAPS_MAX - 2); + goto done; + } + + /* + * The segment created now gets id NXT_PORT_MMAPS_MAX - 1, the last one + * allowed. + */ + + mmap_handler = nxt_port_test_new_port_mmap(task, &mmaps, 1); + + if (mmap_handler == NULL || mmaps.size != NXT_PORT_MMAPS_MAX + || mmap_handler->hdr->id != NXT_PORT_MMAPS_MAX - 1 + || mmaps.elts[NXT_PORT_MMAPS_MAX - 1].mmap_handler != mmap_handler) + { + nxt_log_alert(thr->log, "port mmaps max test: segment %uD was not " + "created", NXT_PORT_MMAPS_MAX - 1); + goto done; + } + + /* The next one would get the limit as its id: refused, nothing grows. */ + + mmap_handler = nxt_port_test_new_port_mmap(task, &mmaps, 1); + + if (mmap_handler != NULL || mmaps.size != NXT_PORT_MMAPS_MAX) { + nxt_log_alert(thr->log, "port mmaps max test: segment %uD was " + "created", NXT_PORT_MMAPS_MAX); + goto done; + } + + port_mmap = nxt_port_test_mmap_at(&mmaps, NXT_PORT_MMAPS_MAX); + + if (port_mmap != NULL || mmaps.size != NXT_PORT_MMAPS_MAX) { + nxt_log_alert(thr->log, "port mmaps max test: id %uD was accepted " + "into a full array", NXT_PORT_MMAPS_MAX); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port mmaps max test passed"); + + ret = NXT_OK; + +done: + + nxt_port_mmaps_destroy(&mmaps, 1); + + nxt_thread_mutex_destroy(&mmaps.mutex); + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index d9415d073..e9d6b6d25 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -265,6 +265,10 @@ main(int argc, char **argv) return 1; } + if (nxt_port_mmaps_max_test(thr) != NXT_OK) { + return 1; + } + if (nxt_port_ready_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 13fe7f96d..1723873cb 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -76,6 +76,7 @@ nxt_int_t nxt_fd_event_change_test(nxt_thread_t *thr); nxt_int_t nxt_port_use_unless_zero_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_read_test(nxt_thread_t *thr); +nxt_int_t nxt_port_mmaps_max_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); diff --git a/src/test/nxt_unit_msg_test.c b/src/test/nxt_unit_msg_test.c index 83ea7706e..a1566be94 100644 --- a/src/test/nxt_unit_msg_test.c +++ b/src/test/nxt_unit_msg_test.c @@ -7,7 +7,8 @@ * Malformed port messages at libunit's receive side (src/nxt_unit.c), fed * through nxt_unit_test_process_msg(). nxt_unit_mmap_read() used to read a * partial last record past the message, mmap_id 0xFFFFFFFF wrapped the - * lib->incoming index, and a segment of any size was accepted. Cases that + * lib->incoming index, a segment id like 100000000 grew lib->incoming to + * that many slots, and a segment of any size was accepted. Cases that * could crash the old code run in a child; a signal is a failure. * * The request cases put an nxt_unit_request_t into segment 0 the way the @@ -377,6 +378,12 @@ static const nxt_unit_msg_test_segment_t segments[] = { { "segment id 0xFFFFFFFF is refused", PORT_MMAP_SIZE, 0xFFFFFFFF, NXT_UNIT_ERROR }, { "segment id 1000 is accepted", PORT_MMAP_SIZE, 1000, NXT_UNIT_OK }, + { "segment id NXT_PORT_MMAPS_MAX - 1 is accepted", PORT_MMAP_SIZE, + NXT_PORT_MMAPS_MAX - 1, NXT_UNIT_OK }, + { "segment id NXT_PORT_MMAPS_MAX is refused", PORT_MMAP_SIZE, + NXT_PORT_MMAPS_MAX, NXT_UNIT_ERROR }, + { "segment id 100000000 is refused", PORT_MMAP_SIZE, 100000000, + NXT_UNIT_ERROR }, { "segment shorter than PORT_MMAP_SIZE is refused", PORT_MMAP_HEADER_SIZE, 1, NXT_UNIT_ERROR }, { "segment longer than PORT_MMAP_SIZE is refused", @@ -389,6 +396,8 @@ static const struct { nxt_port_mmap_msg_t rec; } bad_records[] = { { "mmap_id 0xFFFFFFFF is refused", { 0xFFFFFFFF, 0, 100 } }, + { "mmap_id NXT_PORT_MMAPS_MAX is refused", { NXT_PORT_MMAPS_MAX, 0, 100 } }, + { "mmap_id 100000000 is refused", { 100000000, 0, 100 } }, { "chunk_id past the data area is refused", { 0, PORT_MMAP_CHUNK_COUNT, 100 } }, { "size past the data area is refused", @@ -1076,7 +1085,10 @@ main(void) NXT_UNIT_ERROR); } - /* Not capped: the router's outgoing segments are unbounded (#172). */ + /* + * Capped at NXT_PORT_MMAPS_MAX, not by the shm limit: the router's + * outgoing segments are not bounded by it (#172). + */ for (i = 0; i < nxt_nitems(segments); i++) { nxt_unit_msg_test_in_child(segments[i].name, From 0f0eada9085abd1b4f83f5386d9c6ce3307357e2 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 22:03:47 +0000 Subject: [PATCH 299/391] port: bound fragment reassembly and pair ports in one place Fragment reassembly (#394). A receiving port keeps every fragment of a message until the last fragment arrives. The sender decides how many fragmented messages it opens and how long each one stays open. A peer that never sends the last fragment, or uses a new stream id for each message, makes the receiver grow without a limit. The port now keeps two counters, frag_streams and frag_size. The limits are 64 streams per port, 128 MiB per stream and 256 MiB in flight per port. A stream that goes past a size limit is refused. A new stream when 64 are open drops the oldest one: the port keeps its streams on a queue in start order. Thus streams that a peer never ends cannot block reassembly for good. To keep out the message of another sender, a peer must open 64 new streams between two fragments of that message, for every message. A kept fragment counts as the buffer it came in, nxt_max(payload, port->max_size), so a stream of empty fragments is also bounded. An mmap fragment also counts an upper bound for each buffer that nxt_port_mmap_read() makes: a record can name zero bytes, and a 16 KiB fragment of empty records counted as one buffer but kept about 1,365. The stream keeps the sum and gives back the same amount when it ends. A refused stream is dropped as a whole, with an alert: its buffers are released, and its later fragments find no stream and are discarded. libunit never fragments, so only Unit's own processes send fragments, one message at a time per port. The largest such message is a configuration from the controller, which is far below the limits. An mmap fragment that nxt_port_mmap_read() turned into no shared memory buffer (empty, a partial record, or a first record that names no segment) left msg->buf at the raw read buffer. The stream kept that buffer, and nxt_port_read_handler() also put it back on port->free_bufs and read the next message into it. Such a fragment is now refused: a first fragment starts no stream, a later fragment drops its stream, and the caller frees the read buffer once. The sender cleared the mmap bit only once per message, so a small plain tail fragment after mmap fragments still had the bit. The bit is now cleared for each fragment. Because one stream can now mix plain and mmap fragments, each buffer of the assembled message is released by its own kind, not by the bit of the last fragment. Port pairing (#425). A port on the port list of a process must have port->process set and hold one reference on the process. nxt_process_port_add() sets these together, and nxt_port_release() removes them together. No production path goes around this. nxt_port_release() asserted that a linked port has a process, and then dropped a reference on it. Release builds remove the assert, so a port linked without a process dereferenced NULL. For such a port, nxt_port_release() now unlinks it and logs an alert instead of a crash. The unlink is still necessary, or the list points into the freed pool. nxt_process_port_add() now refuses a port that is already paired, in all builds. The check was only an nxt_assert(), which release builds remove. A second insert of the same link corrupts the list and takes a second reference on the process. The check is one pointer compare on a path that runs once per port, so it stays in release builds and logs an alert. A debug build still traps the caller with the nxt_assert() next to the refusal. Five test fixtures inserted port->link into process->ports by hand, without port->process and without a reference. One test had to unlink by hand in its teardown so that nxt_port_release() did not see a NULL process. The fixtures now use nxt_process_port_add(). The only raw insert of a port link in the tree is in the port release test, which makes the broken state on purpose to check the release guard. The port ready fixtures let nxt_port_release() unlink the port and drop the process reference, and each fixture process has its own hold, as in the wedge test. Tests: - src/test/nxt_port_frag_test.c sends fragment streams past each limit to the reader of a port; two new streams when 64 never end, which drop the two oldest; mmap fragments of 0, 5 and 12 bytes in first, middle and last position; and fragments of empty records until the stream is dropped. - src/test/nxt_port_release_test.c releases a hand-linked port without a process, in a child. It also checks the pairing through the public path: after nxt_process_port_add(), the port has ->process, is on the list, and the process has one more reference. A second add (release builds only) changes none of this; both link neighbours are checked, because a second insert points link.prev at the link itself. The release removes all of it. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 10 + auto/sources | 2 + docs/changes.xml | 13 + src/nxt_port.c | 20 +- src/nxt_port.h | 38 ++ src/nxt_port_socket.c | 300 ++++++++++- src/nxt_process.c | 23 +- src/test/nxt_port_frag_test.c | 640 +++++++++++++++++++++++ src/test/nxt_port_ready_test.c | 80 +-- src/test/nxt_port_release_test.c | 148 ++++++ src/test/nxt_proto_creating_wedge_test.c | 19 +- src/test/nxt_tests.c | 3 +- src/test/nxt_tests.h | 2 + 13 files changed, 1237 insertions(+), 61 deletions(-) create mode 100644 src/test/nxt_port_frag_test.c create mode 100644 src/test/nxt_port_release_test.c diff --git a/CHANGES b/CHANGES index 6855e8030..aaaa851dd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Security: a port kept every fragment of a message until the last + one arrived, so a peer process could make it hold memory without + limit. A fragment that named a shared memory segment but carried no + record also left a receive buffer in the stream while the same + buffer was reused for the next read. Reassembly is now limited to + 64 streams, 128 MiB per stream and 256 MiB per port, a new stream + drops the oldest one when 64 are open, and such a fragment is + dropped. A port linked without a process no longer crashes a + release build when it is freed. + *) Bugfix: an application used a shared memory segment id from the other process to grow its array of segments without a limit. An id like 100000000 in one message made it allocate and fill an array of diff --git a/auto/sources b/auto/sources index dca3c776f..4ffd0fa92 100644 --- a/auto/sources +++ b/auto/sources @@ -212,6 +212,8 @@ NXT_TEST_SRCS=" \ src/test/nxt_conn_close_test.c \ src/test/nxt_router_prepare_msg_test.c \ src/test/nxt_router_response_parse_test.c \ + src/test/nxt_port_frag_test.c \ + src/test/nxt_port_release_test.c \ " diff --git a/docs/changes.xml b/docs/changes.xml index 6156d72fd..c380e198c 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,19 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +a port kept every fragment of a message until the last one arrived, so a +peer process could make it hold memory without limit. A fragment that named +a shared memory segment but carried no record also left a receive buffer in +the stream while the same buffer was reused for the next read. Reassembly +is now limited to 64 streams, 128 MiB per stream and 256 MiB per port, a new +stream drops the oldest one when 64 are open, and such a fragment is dropped. +A port linked without a process no longer crashes a release build when it is +freed. + + + an application used a shared memory segment id from the other process to grow diff --git a/src/nxt_port.c b/src/nxt_port.c index b05e1b598..b06c27020 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -137,6 +137,7 @@ nxt_port_new(nxt_task_t *task, nxt_port_id_t id, nxt_pid_t pid, nxt_mp_cleanup(mp, nxt_port_mp_cleanup, task, port, mp); nxt_queue_init(&port->messages); + nxt_queue_init(&port->frag_queue); nxt_thread_mutex_create(&port->write_mutex); port->queue_fd = -1; @@ -274,8 +275,6 @@ nxt_port_release(nxt_task_t *task, nxt_port_t *port) } if (port->link.next != NULL) { - nxt_assert(port->process != NULL); - rt = task->thread->runtime; /* @@ -297,7 +296,22 @@ nxt_port_release(nxt_task_t *task, nxt_port_t *port) nxt_thread_mutex_unlock(&rt->processes_mutex); - nxt_process_use(task, port->process, -1); + /* + * nxt_process_port_add() is the only production path that links a + * port, and it sets ->process and takes the reference with it. A + * port linked any other way (#425) has no reference to drop: this + * used to be an nxt_assert(), which release builds compile out, and + * nxt_process_use() then dereferenced NULL. The unlink above still + * has to happen -- the list would otherwise point into the pool + * freed below. + */ + if (nxt_fast_path(port->process != NULL)) { + nxt_process_use(task, port->process, -1); + + } else { + nxt_alert(task, "port %p %d:%d was linked to a process without " + "a process reference", port, port->pid, port->id); + } } nxt_mp_release(port->mem_pool); diff --git a/src/nxt_port.h b/src/nxt_port.h index 4e3f23a51..a6642a036 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -286,6 +286,10 @@ struct nxt_port_recv_msg_s { nxt_port_t *port; nxt_port_msg_t port_msg; size_t size; + /* Of a stream being reassembled: what it holds, nxt_port_frag_cost(). */ + size_t frag_held; + /* Of a stream being reassembled: its place in port->frag_queue. */ + nxt_queue_link_t frag_link; #if (NXT_USE_CMSG_PID) nxt_pid_t cmsg_pid; #endif @@ -374,6 +378,10 @@ typedef struct nxt_app_s nxt_app_t; struct nxt_port_s { nxt_fd_event_t socket; + /* + * Set only by nxt_process_port_add(). nxt_port_release() unlinks the + * port and drops the process reference; it does not clear ->process. + */ nxt_queue_link_t link; /* for nxt_process_t.ports */ nxt_process_t *process; @@ -539,6 +547,15 @@ struct nxt_port_s { nxt_lvlhsh_t frags; + /* + * The fragment streams in ->frags and the bytes they hold, kept against + * the NXT_PORT_FRAG_* limits below. Touched only by the port's reader. + */ + uint32_t frag_streams; + uint32_t frag_size; /* <= NXT_PORT_FRAG_TOTAL_MAX */ + /* The same streams, the oldest first. */ + nxt_queue_t frag_queue; + nxt_atomic_t use_count; nxt_process_type_t type; @@ -551,6 +568,27 @@ struct nxt_port_s { }; +/* + * Limits on fragment reassembly at a receiving port (#394). A sender + * controls how many fragmented messages it opens and how long it keeps + * each one going, and the receiver holds every fragment until the last + * one arrives: without a bound, a peer that never sends the last fragment + * -- or sends a new stream id for each message -- grows the receiver + * without limit. libunit never fragments, so the legitimate senders are + * Unit's own processes, one message at a time per destination port; the + * largest such message is a configuration pushed from the controller. + * + * A stream that would pass a size limit is dropped as a whole, with an + * alert: what it had accumulated is released and its later fragments are + * discarded as belonging to no stream. A new stream when the port already + * has NXT_PORT_FRAG_STREAMS_MAX open drops the oldest one the same way, + * so streams that are never ended cannot block the port for good. + */ +#define NXT_PORT_FRAG_STREAMS_MAX 64 /* per port */ +#define NXT_PORT_FRAG_SIZE_MAX (128 * 1024 * 1024) /* per stream */ +#define NXT_PORT_FRAG_TOTAL_MAX (256 * 1024 * 1024) /* per port */ + + typedef struct { nxt_port_id_t id; nxt_pid_t pid; diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 5ba4db8c4..53d8b6609 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -55,6 +55,14 @@ static void nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg); static nxt_buf_t *nxt_port_buf_alloc(nxt_port_t *port); static void nxt_port_buf_free(nxt_port_t *port, nxt_buf_t *b); +static void nxt_port_frag_unaccount(nxt_port_t *port, + nxt_port_recv_msg_t *fmsg); +static void nxt_port_frag_evict(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg); +static void nxt_port_frag_drop(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *fmsg, nxt_bool_t in_hash); +static void nxt_port_frag_bufs_release(nxt_task_t *task, nxt_port_t *port, + nxt_buf_t *b); static nxt_int_t nxt_port_announce(nxt_task_t *task, nxt_port_t *port); static void nxt_port_rearm_now(nxt_task_t *task, nxt_port_t *port); static void nxt_port_retry_later(nxt_task_t *task, nxt_port_t *port); @@ -1136,8 +1144,12 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, /* * Send through mmap enabled only when payload - * is bigger than PORT_MMAP_MIN_SIZE. + * is bigger than PORT_MMAP_MIN_SIZE. The bit is per fragment: a + * fragment sent plain after one sent through mmap must not carry + * it, or the receiver reads its payload as mmap records. */ + msg->port_msg.mmap = 0; + if (m == NXT_PORT_METHOD_MMAP && plain_size > PORT_MMAP_MIN_SIZE) { nxt_port_mmap_write(task, port, msg, &sb, mmsg_buf); @@ -2244,10 +2256,46 @@ static const nxt_lvlhsh_proto_t lvlhsh_frag_proto nxt_aligned(64) = { }; +/* + * An upper bound on what nxt_port_mmap_read() allocates for each record: + * nxt_buf_mem_ts_alloc() takes the memory part of an nxt_buf_t followed by + * its thread-safe completion state, a work item and an engine pointer. + */ +#define NXT_PORT_FRAG_MMAP_BUF_COST \ + (sizeof(nxt_buf_t) + sizeof(nxt_work_t) + sizeof(void *)) + + +/* + * What a fragment kept for reassembly costs the receiver: the whole buffer + * it came in, port->max_size, however little of it carries. Counting the + * payload alone let a stream of empty fragments hold buffers without limit. + * An mmap fragment also keeps a buffer per record, and a record may name + * no bytes at all: a max_size fragment of empty records is some 1,300 + * buffers, so each of them is charged too. + */ +static size_t +nxt_port_frag_cost(nxt_port_t *port, nxt_port_recv_msg_t *msg) +{ + size_t cost; + nxt_buf_t *b; + + cost = nxt_max(msg->size, port->max_size); + + if (msg->port_msg.mmap) { + for (b = msg->buf; b != NULL; b = b->next) { + cost += NXT_PORT_FRAG_MMAP_BUF_COST; + } + } + + return cost; +} + + static nxt_port_recv_msg_t * nxt_port_frag_start(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg) { + size_t cost; nxt_int_t res; nxt_lvlhsh_query_t lhq; nxt_port_recv_msg_t *fmsg; @@ -2255,6 +2303,21 @@ nxt_port_frag_start(nxt_task_t *task, nxt_port_t *port, nxt_debug(task, "start frag stream #%uD", msg->port_msg.stream); + if (nxt_slow_path(port->frag_streams >= NXT_PORT_FRAG_STREAMS_MAX)) { + nxt_port_frag_evict(task, port, msg); + } + + cost = nxt_port_frag_cost(port, msg); + + if (nxt_slow_path(msg->size > NXT_PORT_FRAG_SIZE_MAX + || cost > NXT_PORT_FRAG_TOTAL_MAX - port->frag_size)) + { + nxt_alert(task, "port %d: fragmented message #%uD from pid %PI " + "exceeds the reassembly limit, dropped", port->socket.fd, + msg->port_msg.stream, msg->port_msg.pid); + return NULL; + } + fmsg = nxt_mp_alloc(port->mem_pool, sizeof(nxt_port_recv_msg_t)); if (nxt_slow_path(fmsg == NULL)) { @@ -2279,6 +2342,13 @@ nxt_port_frag_start(nxt_task_t *task, nxt_port_t *port, switch (res) { case NXT_OK: + fmsg->frag_held = cost; + + port->frag_streams++; + port->frag_size += cost; + + nxt_queue_insert_tail(&port->frag_queue, &fmsg->frag_link); + return fmsg; case NXT_DECLINED: @@ -2328,6 +2398,10 @@ nxt_port_frag_find(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg) switch (res) { case NXT_OK: + if (last) { + nxt_port_frag_unaccount(port, lhq.value); + } + return lhq.value; default: @@ -2339,10 +2413,180 @@ nxt_port_frag_find(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg) } +static void +nxt_port_frag_unaccount(nxt_port_t *port, nxt_port_recv_msg_t *fmsg) +{ + port->frag_streams--; + port->frag_size -= fmsg->frag_held; + + nxt_queue_remove(&fmsg->frag_link); +} + + +/* + * Makes room for the new stream in "msg" when NXT_PORT_FRAG_STREAMS_MAX are + * open: the oldest stream is dropped as a whole. A peer that opens streams + * and never ends them thus cannot block reassembly for good. To keep out a + * message of another sender, it must open that many new streams between + * two fragments of that message, for every message. + */ +static void +nxt_port_frag_evict(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg) +{ + nxt_port_recv_msg_t *old; + + old = nxt_queue_link_data(nxt_queue_first(&port->frag_queue), + nxt_port_recv_msg_t, frag_link); + + nxt_alert(task, "port %d: %uD fragmented messages already in progress, " + "dropping the oldest, stream #%uD from pid %PI, for stream " + "#%uD from pid %PI", port->socket.fd, port->frag_streams, + old->port_msg.stream, old->port_msg.pid, msg->port_msg.stream, + msg->port_msg.pid); + + nxt_port_frag_drop(task, port, old, 1); +} + + +/* + * Would appending "msg" to the stream being reassembled in "fmsg" pass a + * limit? The per-port total is checked only for a fragment that keeps the + * stream open, against its "cost": the last one hands the whole message + * over at once. A stream and the port's total are kept under their + * limits, so neither subtraction can wrap. + */ +static nxt_bool_t +nxt_port_frag_fits(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *fmsg, nxt_port_recv_msg_t *msg, size_t cost) +{ + if (nxt_slow_path(msg->size > NXT_PORT_FRAG_SIZE_MAX - fmsg->size)) { + nxt_alert(task, "port %d: fragmented message #%uD from pid %PI " + "exceeds %d bytes, dropped", port->socket.fd, + msg->port_msg.stream, msg->port_msg.pid, + NXT_PORT_FRAG_SIZE_MAX); + return 0; + } + + if (msg->port_msg.mf != 0 + && nxt_slow_path(cost > NXT_PORT_FRAG_TOTAL_MAX - port->frag_size)) + { + nxt_alert(task, "port %d: fragmented messages in progress exceed " + "%d bytes, dropping stream #%uD from pid %PI", + port->socket.fd, NXT_PORT_FRAG_TOTAL_MAX, + msg->port_msg.stream, msg->port_msg.pid); + return 0; + } + + return 1; +} + + +/* + * Gives back the buffers of a reassembled message, each the way it came: + * a shared memory buffer is completed, a read buffer returns to the port. + * The fragments of one stream need not all have come the same way -- the + * sender picks plain or mmap per fragment -- so the last fragment's mmap + * bit does not say how to release what the earlier ones brought. + */ +static void +nxt_port_frag_bufs_release(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) +{ + nxt_buf_t *next; + + for ( /* void */ ; b != NULL; b = next) { + next = b->next; + b->next = NULL; + + if (nxt_buf_is_port_mmap(b)) { + nxt_work_queue_add(port->socket.read_work_queue, + b->completion_handler, task, b, b->parent); + + } else { + nxt_port_buf_free(port, b); + } + } +} + + +/* + * Drops a stream that passed a limit, or the oldest one when a new stream + * needs its place (nxt_port_frag_evict()): out of ->frags if it is still there, + * then everything it had accumulated -- buffers, the first fragment's + * descriptors, the stream itself. Its later fragments find no stream and + * are discarded by the "not found" path. + */ +static void +nxt_port_frag_drop(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *fmsg, nxt_bool_t in_hash) +{ + nxt_lvlhsh_query_t lhq; + nxt_port_frag_key_t frag_key; + + if (in_hash) { + frag_key.stream = fmsg->port_msg.stream; + frag_key.pid = fmsg->port_msg.pid; + + lhq.key_hash = nxt_murmur_hash2(&frag_key, + sizeof(nxt_port_frag_key_t)); + lhq.key.length = sizeof(nxt_port_frag_key_t); + lhq.key.start = (u_char *) &frag_key; + lhq.proto = &lvlhsh_frag_proto; + lhq.pool = port->mem_pool; + + if (nxt_lvlhsh_delete(&port->frags, &lhq) == NXT_OK) { + nxt_port_frag_unaccount(port, fmsg); + } + } + + /* + * A stream started while cancelled holds copies only: the caller freed + * its buffers and closed its descriptors. + */ + if (fmsg->cancelled == 0) { + nxt_port_frag_bufs_release(task, port, fmsg->buf); + + nxt_port_close_fds(fmsg->fd); + } + + nxt_mp_free(port->mem_pool, fmsg); +} + + +/* + * Did nxt_port_mmap_read() turn an mmap fragment into shared memory + * buffers? A fragment that is empty, carries only part of a record, or + * whose first record names no segment leaves msg->buf at the read buffer + * "orig_b", or at NULL. Kept in a stream, the read buffer would also be + * given back to port->free_bufs by the caller and reused for the next + * message while the stream still held it; the sender never sends such a + * fragment, so the fragment is refused. msg->buf is set back to "orig_b" + * for the caller to free. + */ +static nxt_bool_t +nxt_port_frag_mmap_valid(nxt_task_t *task, nxt_port_t *port, + nxt_port_recv_msg_t *msg, nxt_buf_t *orig_b) +{ + if (nxt_fast_path(msg->buf != orig_b && msg->buf != NULL)) { + return 1; + } + + nxt_alert(task, "port %d: mmap fragment of message #%uD from pid %PI " + "carries no shared memory buffer, dropped", port->socket.fd, + msg->port_msg.stream, msg->port_msg.pid); + + msg->buf = orig_b; + msg->size = 0; + + return 0; +} + + static void nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, nxt_port_recv_msg_t *msg) { + size_t cost; nxt_buf_t *b, *orig_b, *next; nxt_port_recv_msg_t *fmsg; @@ -2375,11 +2619,44 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, if (msg->port_msg.mmap) { nxt_port_mmap_read(task, msg); + + if (nxt_slow_path(!nxt_port_frag_mmap_valid(task, port, msg, + orig_b))) + { + /* The last fragment's lookup took it out already. */ + nxt_port_frag_drop(task, port, fmsg, + msg->port_msg.mf != 0); + + b = orig_b; + + goto fmsg_failed; + } + } + + /* The last fragment is not kept, so it is not charged. */ + cost = (msg->port_msg.mf != 0) ? nxt_port_frag_cost(port, msg) : 0; + + if (nxt_slow_path(!nxt_port_frag_fits(task, port, fmsg, msg, + cost))) + { + /* The last fragment's lookup took the stream out already. */ + nxt_port_frag_drop(task, port, fmsg, msg->port_msg.mf != 0); + + /* This fragment's own buffers complete on the common path. */ + b = msg->buf; + + goto fmsg_failed; } nxt_buf_chain_add(&fmsg->buf, msg->buf); fmsg->size += msg->size; + + if (msg->port_msg.mf != 0) { + fmsg->frag_held += cost; + port->frag_size += cost; + } + msg->buf = NULL; b = NULL; @@ -2401,10 +2678,16 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, /* * To disable instant completion or buffer re-usage, - * handler should reset 'msg.buf'. + * handler should reset 'msg.buf'. Whatever it leaves is + * released here, buffer by buffer: this fragment's own read + * buffer is in the chain only when it came plain, and an + * mmap fragment's is freed by the caller once msg->buf is + * restored below. */ - if (!msg->port_msg.mmap && msg->buf == b) { - nxt_port_buf_free(port, b); + if (msg->buf == b) { + nxt_port_frag_bufs_release(task, port, b); + + msg->buf = NULL; } } } @@ -2417,6 +2700,15 @@ nxt_port_read_msg_process(nxt_task_t *task, nxt_port_t *port, if (msg->port_msg.mmap && msg->cancelled == 0) { nxt_port_mmap_read(task, msg); + + if (nxt_slow_path(!nxt_port_frag_mmap_valid(task, port, msg, + orig_b))) + { + b = orig_b; + + goto fmsg_failed; + } + b = msg->buf; } diff --git a/src/nxt_process.c b/src/nxt_process.c index 42a797b8e..08426ba44 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -1356,10 +1356,31 @@ nxt_nanosleep(nxt_nsec_t ns) } +/* + * The only way a port is linked into a process's port list. ->process, + * ->link and the reference the port holds on the process are set together + * here and dropped together by nxt_port_release(), so they cannot diverge + * (#425). A port that is already paired is refused rather than linked a + * second time, in every build: a second insert corrupts the list, and an + * nxt_assert() would compile out of a release build. + */ + void nxt_process_port_add(nxt_task_t *task, nxt_process_t *process, nxt_port_t *port) { - nxt_assert(port->process == NULL); + /* + * A double add is a bug: trap it in a debug build, refuse it in any. + * link.next reads as "already linked" only because a port is never + * linked again after nxt_port_release() unlinks it: in a release build + * nxt_queue_remove() does not clear the link. + */ + nxt_assert(port->process == NULL && port->link.next == NULL); + + if (nxt_slow_path(port->process != NULL || port->link.next != NULL)) { + nxt_alert(task, "port %p %d:%d is already paired with a process", + port, port->pid, port->id); + return; + } port->process = process; nxt_queue_insert_tail(&process->ports, &port->link); diff --git a/src/test/nxt_port_frag_test.c b/src/test/nxt_port_frag_test.c new file mode 100644 index 000000000..2d2bf1a73 --- /dev/null +++ b/src/test/nxt_port_frag_test.c @@ -0,0 +1,640 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Fragment reassembly limits at a receiving port (#394): streams per port, + * bytes per stream and bytes per port. A stream past a limit is dropped + * whole, so its handler never runs. Fragment buffers point into a + * PROT_NONE reservation that nothing reads, so a 128 MB stream is free. + * A fragment counts for the buffer it came in, so empty fragments are + * bounded too. + * + * An mmap fragment that carries no shared memory record -- empty, a partial + * record, or one that names no segment -- is refused: its read buffer used + * to be kept in the stream and also given back to port->free_bufs. + * + * An mmap fragment of empty records keeps a buffer per record while its + * payload is next to nothing; the buffers count too, or a port's total + * admits millions of them. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +#define NXT_FRAG_TEST_MB (1024 * 1024) +#define NXT_FRAG_TEST_STREAM 0x46524700 /* distinct from other tests */ + + +static u_char *nxt_frag_test_space; +static nxt_uint_t nxt_frag_test_calls; +static uint32_t nxt_frag_test_last_stream; +static size_t nxt_frag_test_last_size; +static nxt_bool_t nxt_frag_test_oom; + +static nxt_task_t *nxt_frag_test_task; +static nxt_port_t *nxt_frag_test_port; +static nxt_bool_t nxt_frag_test_aliased; + +static nxt_pid_t nxt_frag_test_sender; +static nxt_chunk_id_t nxt_frag_test_chunk; + + +typedef struct { + size_t size; /* of the mmap fragment's payload */ + nxt_uint_t pos; /* of the mmap fragment in a stream of three */ +} nxt_frag_test_mmap_case_t; + + +static const nxt_frag_test_mmap_case_t nxt_frag_test_mmap_cases[] = { + { 0, 0 }, { 0, 1 }, { 0, 2 }, + { 5, 0 }, { 5, 1 }, { 5, 2 }, + { sizeof(nxt_port_mmap_msg_t), 0 }, + { sizeof(nxt_port_mmap_msg_t), 1 }, + { sizeof(nxt_port_mmap_msg_t), 2 }, +}; + + +static void +nxt_frag_test_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_frag_test_calls++; + nxt_frag_test_last_stream = msg->port_msg.stream; + nxt_frag_test_last_size = msg->size; +} + + +static void +nxt_frag_test_send(nxt_task_t *task, nxt_port_t *port, uint32_t stream, + size_t size, nxt_bool_t first, nxt_bool_t last) +{ + nxt_buf_t *b; + nxt_port_recv_msg_t msg; + + b = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_buf_t)); + if (nxt_slow_path(b == NULL)) { + nxt_frag_test_oom = 1; + return; + } + + b->mem.start = nxt_frag_test_space; + b->mem.pos = nxt_frag_test_space; + b->mem.free = nxt_frag_test_space; + b->mem.end = nxt_frag_test_space + size; + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t) + size; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.stream = stream; + msg.port_msg.pid = nxt_pid; + msg.port_msg.type = _NXT_PORT_MSG_STATUS; + msg.port_msg.nf = !first; + msg.port_msg.mf = !last; + + nxt_port_test_run_read_msg_process(task, port, &msg); +} + + +/* A stream of "n" fragments of "mb" MB: delivered whole, or not at all. */ +static nxt_int_t +nxt_frag_test_expect(nxt_task_t *task, nxt_port_t *port, const char *name, + uint32_t stream, nxt_uint_t n, size_t mb, nxt_bool_t expect) +{ + nxt_uint_t i, calls; + + calls = nxt_frag_test_calls; + + for (i = 0; i < n; i++) { + nxt_frag_test_send(task, port, stream, mb * NXT_FRAG_TEST_MB, i == 0, + i == n - 1); + } + + NXT_TEST_CHECK(task->log, (nxt_frag_test_calls != calls) == expect, + "port frag test: %s: %s", name, + expect ? "dropped" : "delivered"); + NXT_TEST_CHECK(task->log, !expect + || (nxt_frag_test_last_stream == stream + && nxt_frag_test_last_size == n * mb * NXT_FRAG_TEST_MB), + "port frag test: %s: %uz bytes", name, + nxt_frag_test_last_size); + return NXT_OK; +} + + +static nxt_int_t +nxt_frag_test_run(nxt_task_t *task, nxt_port_t *port) +{ + size_t mb, half; + uint32_t s; + nxt_uint_t i, n, calls; + + s = NXT_FRAG_TEST_STREAM; + mb = NXT_PORT_FRAG_SIZE_MAX / NXT_FRAG_TEST_MB; + + /* Per stream: at the limit; past it on the last or a middle fragment. */ + + if (nxt_frag_test_expect(task, port, "at the size limit", s++, mb, 1, 1) + || nxt_frag_test_expect(task, port, "past it, last", s++, mb + 1, 1, 0) + || nxt_frag_test_expect(task, port, "past it, middle", s++, mb + 2, 1, + 0) + || nxt_frag_test_expect(task, port, "after a drop", s++, 3, 1, 1)) + { + return NXT_ERROR; + } + + /* + * Per port, streams: open the most there may be, never end them, then + * two more. Each new stream drops the oldest open one, so streams that + * are never ended cannot block the port. + */ + + for (i = 0; i < NXT_PORT_FRAG_STREAMS_MAX; i++) { + nxt_frag_test_send(task, port, s + i, 1, 1, 0); + } + + calls = nxt_frag_test_calls; + + nxt_frag_test_send(task, port, s + NXT_PORT_FRAG_STREAMS_MAX, 1, 1, 0); + nxt_frag_test_send(task, port, s + NXT_PORT_FRAG_STREAMS_MAX + 1, 1, 1, 0); + + NXT_TEST_CHECK(task->log, + port->frag_streams == NXT_PORT_FRAG_STREAMS_MAX, + "port frag test: %uD streams open after two evictions", + port->frag_streams); + + for (i = 0; i < 2; i++) { + nxt_frag_test_send(task, port, s + NXT_PORT_FRAG_STREAMS_MAX + i, 1, + 0, 1); + + NXT_TEST_CHECK(task->log, nxt_frag_test_calls == calls + i + 1 + && nxt_frag_test_last_stream + == s + NXT_PORT_FRAG_STREAMS_MAX + i, + "port frag test: a stream past the most was refused"); + } + + /* The two oldest are gone: their last fragments find no stream. */ + + calls = nxt_frag_test_calls; + + nxt_frag_test_send(task, port, s, 1, 0, 1); + nxt_frag_test_send(task, port, s + 1, 1, 0, 1); + + NXT_TEST_CHECK(task->log, nxt_frag_test_calls == calls, + "port frag test: an evicted stream completed"); + + /* The others still complete, in any order, and free their places. */ + + for (i = NXT_PORT_FRAG_STREAMS_MAX; i-- > 2; /* void */) { + nxt_frag_test_send(task, port, s + i, 1, 0, 1); + } + + NXT_TEST_CHECK(task->log, + nxt_frag_test_calls == calls + NXT_PORT_FRAG_STREAMS_MAX - 2 + && port->frag_streams == 0 && port->frag_size == 0, + "port frag test: %uD of the open streams completed, " + "%uD left", nxt_frag_test_calls - calls, + port->frag_streams); + + s += NXT_PORT_FRAG_STREAMS_MAX + 2; + + /* Per port, bytes: two open halves fill it; a third stream is dropped. */ + + half = NXT_PORT_FRAG_TOTAL_MAX / 2 / NXT_FRAG_TEST_MB * NXT_FRAG_TEST_MB; + + nxt_frag_test_send(task, port, s, half, 1, 0); + nxt_frag_test_send(task, port, s + 1, half, 1, 0); + + if (nxt_frag_test_expect(task, port, "past the port's total", s + 2, 2, + 1, 0)) + { + return NXT_ERROR; + } + + calls = nxt_frag_test_calls; + + nxt_frag_test_send(task, port, s, 0, 0, 1); + nxt_frag_test_send(task, port, s + 1, 0, 0, 1); + + NXT_TEST_CHECK(task->log, nxt_frag_test_calls == calls + 2, + "port frag test: streams within the total did not " + "complete"); + + /* Everything is accounted back: a fresh stream at the limit passes. */ + + if (nxt_frag_test_expect(task, port, "at the limit, again", s + 3, mb, 1, + 1)) + { + return NXT_ERROR; + } + + /* + * Empty fragments: each one holds a buffer of port->max_size, so the + * port's total admits NXT_PORT_FRAG_TOTAL_MAX / max_size of them. + */ + + port->max_size = 16 * 1024; + n = NXT_PORT_FRAG_TOTAL_MAX / port->max_size; + + if (nxt_frag_test_expect(task, port, "empty fragments up to the total", + s + 4, n + 1, 0, 1) + || nxt_frag_test_expect(task, port, "empty fragments past the total", + s + 5, n + 2, 0, 0)) + { + return NXT_ERROR; + } + + NXT_TEST_CHECK(task->log, !nxt_frag_test_oom, + "port frag test: out of memory"); + return NXT_OK; +} + + +/* Would the assembled message hand over a buffer the port already reuses? */ +static void +nxt_frag_test_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_buf_t *b, *f; + + nxt_frag_test_calls++; + + for (b = msg->buf; b != NULL; b = b->next) { + for (f = msg->port->free_bufs; f != NULL; f = f->next) { + if (f == b) { + nxt_frag_test_aliased = 1; + } + } + } +} + + +/* + * Sends one fragment the way nxt_port_read_handler() receives it: in a + * buffer of its own, given back to port->free_bufs afterwards unless the + * dispatch took it. + */ +static nxt_int_t +nxt_frag_test_mmap_send(nxt_port_t *port, const u_char *data, size_t size, + nxt_bool_t mmap, nxt_bool_t first, nxt_bool_t last) +{ + nxt_buf_t *b; + nxt_port_recv_msg_t msg; + + b = nxt_buf_mem_alloc(port->mem_pool, 64, 0); + if (b == NULL) { + return NXT_ERROR; + } + + nxt_memcpy(b->mem.start, data, size); + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t) + size; + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.stream = NXT_FRAG_TEST_STREAM - 1; + msg.port_msg.pid = nxt_pid; + msg.port_msg.type = _NXT_PORT_MSG_STATUS; + msg.port_msg.nf = !first; + msg.port_msg.mf = !last; + msg.port_msg.mmap = mmap; + + nxt_port_test_run_read_msg_process(nxt_frag_test_task, port, &msg); + + if (msg.buf == b) { + b->next = port->free_bufs; + port->free_bufs = b; + } + + return NXT_OK; +} + + +/* In a child: the last position used to complete plain buffers as mmap. */ +static int +nxt_frag_test_mmap_child(void *data) +{ + u_char payload[64]; + uint32_t streams, size; + nxt_uint_t i; + nxt_port_t *port; + nxt_port_mmap_msg_t rec; + const nxt_frag_test_mmap_case_t *tc; + + tc = data; + port = nxt_frag_test_port; + + /* A whole record, for a segment the sender never shared. */ + rec.mmap_id = 0; + rec.chunk_id = 0; + rec.size = 100; + + nxt_memset(payload, 'x', sizeof(payload)); + nxt_memcpy(payload, &rec, sizeof(rec)); + + port->handler = nxt_frag_test_mmap_handler; + nxt_frag_test_calls = 0; + nxt_frag_test_aliased = 0; + + streams = port->frag_streams; + size = port->frag_size; + + for (i = 0; i < 3; i++) { + if (nxt_frag_test_mmap_send(port, payload, + (i == tc->pos) ? tc->size : 16, + i == tc->pos, i == 0, i == 2) + != NXT_OK) + { + return 8; + } + } + + /* 1: delivered; 2: with a buffer the port reuses; 4: not given back. */ + return (nxt_frag_test_calls != 0) + | (nxt_frag_test_aliased << 1) + | ((port->frag_streams != streams || port->frag_size != size) << 2); +} + + +static nxt_int_t +nxt_frag_test_mmap_run(nxt_thread_t *thr, nxt_port_t *port) +{ + int status; + nxt_int_t ret; + nxt_uint_t i; + nxt_runtime_t *rt, *saved_rt; + + /* A runtime that knows no sender, so no record names a segment. */ + rt = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_runtime_t)); + if (rt == NULL + || nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK) + { + return NXT_ERROR; + } + + saved_rt = thr->runtime; + thr->runtime = rt; + + nxt_frag_test_task = thr->task; + nxt_frag_test_port = port; + + ret = NXT_OK; + + for (i = 0; i < nxt_nitems(nxt_frag_test_mmap_cases); i++) { + status = nxt_test_in_child(thr, "port frag test", + nxt_frag_test_mmap_child, + (void *) &nxt_frag_test_mmap_cases[i]); + if (status != 0) { + nxt_log_alert(thr->log, "port frag test: an mmap fragment of " + "%uz bytes at position %ui was not refused (%d)", + nxt_frag_test_mmap_cases[i].size, + nxt_frag_test_mmap_cases[i].pos, status); + ret = NXT_ERROR; + } + } + + thr->runtime = saved_rt; + + nxt_thread_mutex_destroy(&rt->processes_mutex); + + return ret; +} + + +/* + * In a child: with the port's total all taken but "room", open a stream of + * mmap fragments, each a max_size read buffer full of empty records for a + * segment the sender did share, and send until the stream is dropped. + * Every record keeps a buffer of at least sizeof(nxt_buf_t), so what the + * stream held must fit in the room it had. + */ +static int +nxt_frag_test_records_child(void *data) +{ + size_t room, nrecs; + uint32_t streams; + nxt_buf_t *b; + nxt_uint_t i, j, n; + nxt_port_t *port; + nxt_work_queue_t wq; + nxt_port_mmap_msg_t *rec; + nxt_port_recv_msg_t msg; + nxt_work_queue_cache_t cache; + + port = nxt_frag_test_port; + port->handler = nxt_frag_test_handler; + port->max_size = 16 * 1024; + + /* Where a dropped stream's shared memory buffers complete. */ + nxt_memzero(&wq, sizeof(nxt_work_queue_t)); + nxt_work_queue_cache_create(&cache, 1024); + wq.cache = &cache; + port->socket.read_work_queue = &wq; + + nrecs = port->max_size / sizeof(nxt_port_mmap_msg_t); + room = 64 * port->max_size; + n = room / port->max_size + 2; + + streams = port->frag_streams; + port->frag_size = NXT_PORT_FRAG_TOTAL_MAX - room; + + for (i = 0; i < n; i++) { + b = nxt_buf_mem_alloc(port->mem_pool, port->max_size, 0); + if (b == NULL) { + return 8; + } + + rec = (nxt_port_mmap_msg_t *) b->mem.start; + + for (j = 0; j < nrecs; j++) { + rec[j].mmap_id = 0; + rec[j].chunk_id = nxt_frag_test_chunk; + rec[j].size = 0; + } + + nxt_memzero(&msg, sizeof(nxt_port_recv_msg_t)); + + msg.port = port; + msg.buf = b; + msg.size = sizeof(nxt_port_msg_t) + nrecs * sizeof(*rec); + msg.fd[0] = -1; + msg.fd[1] = -1; + msg.port_msg.stream = NXT_FRAG_TEST_STREAM - 2; + msg.port_msg.pid = nxt_frag_test_sender; + msg.port_msg.type = _NXT_PORT_MSG_STATUS; + msg.port_msg.nf = (i != 0); + msg.port_msg.mf = 1; + msg.port_msg.mmap = 1; + + nxt_port_test_run_read_msg_process(nxt_frag_test_task, port, &msg); + + if (msg.buf == b) { + b->next = port->free_bufs; + port->free_bufs = b; + } + + if (port->frag_streams == streams) { + break; + } + } + + /* 1: held more than its room; 2: never dropped; 4: not given back. */ + return (i * nrecs * sizeof(nxt_buf_t) > room) + | ((i == n) << 1) + | ((port->frag_size != NXT_PORT_FRAG_TOTAL_MAX - room) << 2); +} + + +/* + * The runtime knows one sender, with a segment in its incoming array, so + * nxt_port_mmap_read() turns each of its records into a buffer. + */ +static nxt_int_t +nxt_frag_test_records_run(nxt_thread_t *thr, nxt_port_t *port) +{ + int status; + nxt_buf_t *seg; + nxt_task_t *task; + nxt_process_t *process; + nxt_runtime_t *rt, *saved_rt; + nxt_event_engine_t engine, *saved_engine; + nxt_port_mmap_handler_t *mmap_handler; + + task = thr->task; + + rt = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_runtime_t)); + process = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_process_t)); + + if (rt == NULL || process == NULL + || nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK) + { + return NXT_ERROR; + } + + rt->mem_pool = port->mem_pool; + + nxt_memzero(&engine, sizeof(engine)); + engine.mem_pool = port->mem_pool; + engine.task.thread = thr; + engine.task.log = thr->log; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + thr->runtime = rt; + thr->engine = &engine; + + status = -1; + + process->pid = nxt_pid + 43; + process->use_count = 1; + nxt_queue_init(&process->ports); + + if (nxt_thread_mutex_create(&process->incoming.mutex) != NXT_OK) { + goto done; + } + + nxt_runtime_process_add(task, process); + + seg = nxt_port_mmap_get_buf(task, &process->incoming, + PORT_MMAP_CHUNK_SIZE); + if (seg == NULL) { + goto done; + } + + mmap_handler = seg->parent; + + nxt_frag_test_task = task; + nxt_frag_test_port = port; + nxt_frag_test_sender = process->pid; + nxt_frag_test_chunk = nxt_port_mmap_chunk_id(mmap_handler->hdr, + seg->mem.pos); + + status = nxt_test_in_child(thr, "port frag test", + nxt_frag_test_records_child, NULL); + if (status != 0) { + nxt_log_alert(thr->log, "port frag test: mmap fragments of empty " + "records passed the port's total (%d)", status); + } + +done: + + /* The segment stays mapped, like other port fixtures. */ + + thr->runtime = saved_rt; + thr->engine = saved_engine; + + return (status == 0) ? NXT_OK : NXT_ERROR; +} + + +nxt_int_t +nxt_port_frag_test(nxt_thread_t *thr) +{ + size_t reserve; + nxt_int_t ret; + nxt_task_t *task; + nxt_port_t *port; + + nxt_thread_time_update(thr); + + task = thr->task; + task->thread = thr; + + reserve = NXT_PORT_FRAG_TOTAL_MAX + 16 * NXT_FRAG_TEST_MB; + + nxt_frag_test_space = mmap(NULL, reserve, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (nxt_frag_test_space == MAP_FAILED) { + nxt_log_alert(thr->log, "port frag test: mmap() failed %E", nxt_errno); + return NXT_ERROR; + } + + port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(port == NULL)) { + (void) munmap(nxt_frag_test_space, reserve); + return NXT_ERROR; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + port->handler = nxt_frag_test_handler; + + ret = nxt_frag_test_run(task, port); + + if (ret == NXT_OK) { + ret = nxt_frag_test_mmap_run(thr, port); + } + + if (ret == NXT_OK) { + ret = nxt_frag_test_records_run(thr, port); + } + + /* + * The port's pool holds everything the streams allocated, the buffers + * on port->free_bufs included; nothing else refers to the port. + */ + nxt_port_use(task, port, -1); + + (void) munmap(nxt_frag_test_space, reserve); + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port frag test passed"); + } + + return ret; +} diff --git a/src/test/nxt_port_ready_test.c b/src/test/nxt_port_ready_test.c index 3dda9d6d3..618aefc9a 100644 --- a/src/test/nxt_port_ready_test.c +++ b/src/test/nxt_port_ready_test.c @@ -1017,7 +1017,10 @@ nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, proto_port->pair[1] = -1; proto_port->socket.fd = -1; - nxt_queue_insert_tail(&proto->ports, &proto_port->link); + /* The fixture's own hold on the record, as main's would be. */ + proto->use_count = 1; + + nxt_process_port_add(task, proto, proto_port); /* * A second peer, of main's type. nxt_proc_send_matrix[] pairs MAIN with @@ -1042,7 +1045,10 @@ nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, main_port->pair[1] = -1; main_port->socket.fd = -1; - nxt_queue_insert_tail(&mainp->ports, &main_port->link); + /* The fixture's own hold on the record, as main's would be. */ + mainp->use_count = 1; + + nxt_process_port_add(task, mainp, main_port); router->pid = nxt_pid + 9; router->isolated_pid = nxt_pid + 9; @@ -1060,7 +1066,10 @@ nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, router_port->pair[1] = -1; router_port->socket.fd = -1; - nxt_queue_insert_tail(&router->ports, &router_port->link); + /* The fixture's own hold on the record, as main's would be. */ + router->use_count = 1; + + nxt_process_port_add(task, router, router_port); /* * An engine only for the announcement: nxt_port_send_port() allocates @@ -1304,16 +1313,15 @@ nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, done: /* - * Each port was linked onto its process's queue directly, without - * nxt_process_port_add(), so port->process was never set and - * nxt_port_release() must not be handed a port still on that queue -- - * it would assert. Unlinking first and dropping the reference with - * nxt_port_use() frees the port's own pool (see nxt_port_new()), which - * the caller's nxt_mp_destroy(mp) cannot reach. proto/mainp/router - * need no matching release: they come from mp and are freed along with - * it, but nxt_runtime_process_add() put them in rt->processes, a - * malloc'd table outside mp, and nxt_runtime_process_remove() is what - * takes them back out. + * Each port was paired with its process by nxt_process_port_add(), + * which took a process reference. Dropping the port's last reference + * with nxt_port_use() lets nxt_port_release() unlink it, drop that + * process reference (2 -> 1, the fixture keeps its own hold) and free + * the port's own pool (see nxt_port_new()), which the caller's + * nxt_mp_destroy(mp) cannot reach. proto/mainp/router come from mp and + * are freed along with it, but nxt_runtime_process_add() put them in + * rt->processes, a malloc'd table outside mp, and + * nxt_runtime_process_remove() is what takes them back out. * * That add happens for each of the three only partway through the * function -- main_port is created before router is registered -- so @@ -1324,22 +1332,16 @@ nxt_port_ready_test_stream(nxt_thread_t *thr, nxt_task_t *task, */ if (main_port != NULL) { nxt_port_close(task, main_port); - nxt_queue_remove(&main_port->link); - main_port->link.next = NULL; nxt_port_use(task, main_port, -1); } if (router_port != NULL) { nxt_port_close(task, router_port); - nxt_queue_remove(&router_port->link); - router_port->link.next = NULL; nxt_port_use(task, router_port, -1); } if (proto_port != NULL) { nxt_port_close(task, proto_port); - nxt_queue_remove(&proto_port->link); - proto_port->link.next = NULL; nxt_port_use(task, proto_port, -1); } @@ -1504,7 +1506,10 @@ nxt_port_ready_test(nxt_thread_t *thr) port->pair[1] = -1; port->socket.fd = -1; - nxt_queue_insert_tail(&process->ports, &port->link); + /* The fixture's own hold on the record, as main's would be. */ + process->use_count = 1; + + nxt_process_port_add(task, process, port); #if (NXT_USE_CMSG_PID) @@ -1597,7 +1602,10 @@ nxt_port_ready_test(nxt_thread_t *thr) queueless_port->pair[1] = -1; queueless_port->socket.fd = -1; - nxt_queue_insert_tail(&queueless->ports, &queueless_port->link); + /* The fixture's own hold on the record, as main's would be. */ + queueless->use_count = 1; + + nxt_process_port_add(task, queueless, queueless_port); msg.port_msg.pid = queueless->pid; @@ -1640,7 +1648,10 @@ nxt_port_ready_test(nxt_thread_t *thr) nofd_port->pair[1] = -1; nofd_port->socket.fd = -1; - nxt_queue_insert_tail(&nofd->ports, &nofd_port->link); + /* The fixture's own hold on the record, as main's would be. */ + nofd->use_count = 1; + + nxt_process_port_add(task, nofd, nofd_port); msg.port_msg.pid = nofd->pid; @@ -1690,35 +1701,28 @@ nxt_port_ready_test(nxt_thread_t *thr) * and nothing else owns them here: releasing the pool would leak both. * The fixture port never had a socket pair, so this only frees the queue. * - * Each port was linked onto its process's queue directly, without - * nxt_process_port_add(), so port->process was never set and - * nxt_port_release() must not be handed a port still on that queue -- - * it would assert. Unlinking first and dropping the reference with - * nxt_port_use() frees the port's own pool (see nxt_port_new()), which - * nxt_mp_destroy(mp) below cannot reach. The process itself needs no - * matching release: it came from mp and is freed along with it, but - * nxt_runtime_process_add() put it in rt->processes, a malloc'd table - * outside mp, and nxt_runtime_process_remove() is what takes it back - * out. + * Each port was paired with its process by nxt_process_port_add(), + * which took a process reference. Dropping the port's last reference + * with nxt_port_use() lets nxt_port_release() unlink it, drop that + * process reference (2 -> 1, the fixture keeps its own hold) and free + * the port's own pool (see nxt_port_new()), which nxt_mp_destroy(mp) + * below cannot reach. The process itself came from mp and is freed + * along with it, but nxt_runtime_process_add() put it in rt->processes, + * a malloc'd table outside mp, and nxt_runtime_process_remove() is what + * takes it back out. */ if (port != NULL) { nxt_port_close(task, port); - nxt_queue_remove(&port->link); - port->link.next = NULL; nxt_port_use(task, port, -1); } if (queueless_port != NULL) { nxt_port_close(task, queueless_port); - nxt_queue_remove(&queueless_port->link); - queueless_port->link.next = NULL; nxt_port_use(task, queueless_port, -1); } if (nofd_port != NULL) { nxt_port_close(task, nofd_port); - nxt_queue_remove(&nofd_port->link); - nofd_port->link.next = NULL; nxt_port_use(task, nofd_port, -1); } diff --git a/src/test/nxt_port_release_test.c b/src/test/nxt_port_release_test.c new file mode 100644 index 000000000..d4c4dcada --- /dev/null +++ b/src/test/nxt_port_release_test.c @@ -0,0 +1,148 @@ + +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * The port/process pairing (#425). Through nxt_process_port_add(), the only + * public path, a linked port always has ->process and one reference on it, + * a second add changes nothing, and nxt_port_release() takes all of it + * back. A port linked into a process's list by hand, with no ->process and + * no reference, used to abort a debug build and dereference NULL in a + * release build: it must be unlinked, and no reference it does not hold + * may be dropped. Runs in a child, so a crash is a failure. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static int +nxt_port_release_test_child(void *data) +{ + nxt_task_t *task; + nxt_port_t *port, *paired; + nxt_thread_t *thr; + nxt_runtime_t *rt; + nxt_process_t *process; + + rt = data; + thr = nxt_thread(); + thr->runtime = rt; + thr->engine = NULL; + task = thr->task; + + process = nxt_mp_zalloc(rt->mem_pool, sizeof(nxt_process_t)); + if (process == NULL) { + return 2; + } + + process->pid = nxt_pid + 53; + process->use_count = 1; + nxt_queue_init(&process->ports); + + nxt_runtime_process_add(task, process); + + /* A correctly paired port first: its release drops its reference. */ + + paired = nxt_port_new(task, 1, process->pid, NXT_PROCESS_APP); + if (paired == NULL) { + return 2; + } + + paired->pair[0] = -1; + paired->pair[1] = -1; + paired->socket.fd = -1; + + nxt_process_port_add(task, process, paired); + + if (paired->process != process + || nxt_queue_first(&process->ports) != &paired->link + || process->use_count != 2) + { + return 3; + } + +#if !(NXT_DEBUG) + + /* Already paired: refused in a release build; a debug build asserts. */ + + nxt_process_port_add(task, process, paired); + + if (paired->process != process + || nxt_queue_next(&paired->link) != nxt_queue_tail(&process->ports) + || nxt_queue_prev(&paired->link) != nxt_queue_head(&process->ports) + || process->use_count != 2) + { + return 7; + } + +#endif + + nxt_port_use(task, paired, -1); + + if (process->use_count != 1 || !nxt_queue_is_empty(&process->ports)) { + return 4; + } + + /* Linked by hand: no port->process, no reference. */ + + port = nxt_port_new(task, 2, process->pid, NXT_PROCESS_APP); + if (port == NULL) { + return 2; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + nxt_queue_insert_tail(&process->ports, &port->link); + + nxt_port_use(task, port, -1); + + /* A port still on the list would point into the freed pool. */ + return !nxt_queue_is_empty(&process->ports) ? 5 + : process->use_count != 1 ? 6 : 0; +} + + +nxt_int_t +nxt_port_release_test(nxt_thread_t *thr) +{ + int rc; + nxt_mp_t *mp; + nxt_runtime_t *rt; + + thr->task->thread = thr; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + + if (nxt_slow_path(rt == NULL + || nxt_thread_mutex_create(&rt->processes_mutex) + != NXT_OK)) + { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + rt->mem_pool = mp; + + rc = nxt_test_in_child(thr, "port release test", + nxt_port_release_test_child, rt); + + nxt_mp_destroy(mp); + + NXT_TEST_CHECK(thr->log, rc == 0, "port release test failed at step %d", + rc); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "port release test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_proto_creating_wedge_test.c b/src/test/nxt_proto_creating_wedge_test.c index 490262b02..10c861301 100644 --- a/src/test/nxt_proto_creating_wedge_test.c +++ b/src/test/nxt_proto_creating_wedge_test.c @@ -289,7 +289,10 @@ nxt_proto_creating_wedge_test(nxt_thread_t *thr) app_port->pair[1] = -1; app_port->socket.fd = -1; - nxt_queue_insert_tail(&process->ports, &app_port->link); + /* The fixture's own hold on the record, as main's would be. */ + process->use_count = 1; + + nxt_process_port_add(task, process, app_port); /* * A child that died before PROCESS_CREATED, with the prototype in main's @@ -572,20 +575,8 @@ nxt_proto_creating_wedge_test(nxt_thread_t *thr) nxt_runtime_port_remove(task, main_port); } + /* Paired through nxt_process_port_add(), so its release unlinks it. */ if (app_port != NULL) { - nxt_queue_remove(&app_port->link); - - /* - * nxt_queue_remove() nulls the link only in a debug build - * (src/nxt_queue.h:130-148). nxt_port_release() reads a non-NULL - * link.next as "this port belongs to a process" and dereferences - * port->process, which nxt_port_new() never set -- guarded by an - * nxt_assert() that compiles out of a release build, so the test - * would segfault in teardown there and nowhere else. - */ - app_port->link.next = NULL; - app_port->link.prev = NULL; - nxt_port_use(task, app_port, -1); } diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index e9d6b6d25..4eba1b5eb 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -72,7 +72,8 @@ nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), static nxt_int_t (*const nxt_security_tests[])(nxt_thread_t *) = { nxt_checked_test, nxt_port_mmap_read_test, - nxt_router_response_parse_test, + nxt_router_response_parse_test, nxt_port_frag_test, + nxt_port_release_test, }; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 1723873cb..ce63a26a7 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -102,6 +102,8 @@ nxt_int_t nxt_port_queued_fd_test(nxt_thread_t *thr); nxt_int_t nxt_checked_test(nxt_thread_t *thr); nxt_int_t nxt_conn_close_test(nxt_thread_t *thr); nxt_int_t nxt_router_response_parse_test(nxt_thread_t *thr); +nxt_int_t nxt_port_frag_test(nxt_thread_t *thr); +nxt_int_t nxt_port_release_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From b040963420a4b603f0defed35eb0b770823af5bd Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 24 Sep 2026 17:57:52 +0000 Subject: [PATCH 300/391] fix(port): bound the retries of the shared-memory queues The port queue (nxt_port_queue.h) and the application queue (nxt_app_queue.h) use the circular queues of nxt_nncq.h and nxt_app_nncq.h. Both queues are in shared memory. The application maps this memory writable. nxt_nncq_enqueue() and nxt_nncq_dequeue() retry until the entry under the tail or the head has a cycle they can use. A peer can write a different cycle into that entry. Then the router retried that entry for ever. The nxt_app_nncq_* functions have the same problem. Each loop now has a retry budget of one queue size. Normal contention needs much less than that: each retry means that another party made progress. When the budget is used up, an enqueue returns NXT_ERROR and a dequeue reports the queue empty. The callers handle the new error: - nxt_port_queue_send() and nxt_app_queue_send() put the slot they took back into free_items and return NXT_ERROR. The slot is not lost with the message. - nxt_port_socket_write2() writes an alert and returns NXT_ERROR to its caller, as for a send to a dead peer. The caller still owns the buffers and the descriptors. The port keeps the queue because other engines send on it. Each later message meets the same retry budget, so each one costs at most one pass over the queue. Test: src/test/nxt_nncq_bound_test.c poisons one queue for each case and runs the router-side operation in a child process under alarm(). The operation must return and report the queue empty or the send failed. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 5 + auto/sources | 1 + docs/changes.xml | 9 ++ src/nxt_app_nncq.h | 23 +++-- src/nxt_app_queue.h | 8 +- src/nxt_nncq.h | 28 +++-- src/nxt_port_queue.h | 9 +- src/nxt_port_socket.c | 25 +++++ src/test/nxt_nncq_bound_test.c | 182 +++++++++++++++++++++++++++++++++ src/test/nxt_tests.c | 2 +- src/test/nxt_tests.h | 1 + 11 files changed, 270 insertions(+), 23 deletions(-) create mode 100644 src/test/nxt_nncq_bound_test.c diff --git a/CHANGES b/CHANGES index aaaa851dd..b8487aaf6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Security: an application could write an unexpected value into an + entry of a shared memory queue. The router then retried that entry + for ever. Each enqueue and dequeue now stops after one queue size + of retries. The router refuses the message and writes an alert. + *) Security: a port kept every fragment of a message until the last one arrived, so a peer process could make it hold memory without limit. A fragment that named a shared memory segment but carried no diff --git a/auto/sources b/auto/sources index 4ffd0fa92..e41527f7a 100644 --- a/auto/sources +++ b/auto/sources @@ -214,6 +214,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_response_parse_test.c \ src/test/nxt_port_frag_test.c \ src/test/nxt_port_release_test.c \ + src/test/nxt_nncq_bound_test.c \ " diff --git a/docs/changes.xml b/docs/changes.xml index c380e198c..6ec7ad632 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,15 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +an application could write an unexpected value into an entry of a shared +memory queue. The router then retried that entry for ever. Each enqueue and +dequeue now stops after one queue size of retries. The router refuses the +message and writes an alert. + + + a port kept every fragment of a message until the last one arrived, so a diff --git a/src/nxt_app_nncq.h b/src/nxt_app_nncq.h index f9b8ce0c8..38414e4df 100644 --- a/src/nxt_app_nncq.h +++ b/src/nxt_app_nncq.h @@ -11,6 +11,9 @@ #define NXT_APP_NNCQ_SIZE 131072 +/* The retry budget of the loops below, see nxt_nncq.h. */ +#define NXT_APP_NNCQ_RETRIES NXT_APP_NNCQ_SIZE + typedef uint32_t nxt_app_nncq_atomic_t; typedef uint16_t nxt_app_nncq_cycle_t; @@ -97,13 +100,13 @@ nxt_app_nncq_init(nxt_app_nncq_t volatile *q) } -static void +static nxt_int_t nxt_app_nncq_enqueue(nxt_app_nncq_t volatile *q, nxt_app_nncq_atomic_t val) { nxt_app_nncq_cycle_t e_cycle, t_cycle; - nxt_app_nncq_atomic_t n, t, e, j; + nxt_app_nncq_atomic_t n, t, e, j, r; - for ( ;; ) { + for (r = 0; r < NXT_APP_NNCQ_RETRIES; r++) { t = nxt_app_nncq_tail(q); j = nxt_app_nncq_map(q, t); e = q->entries[j]; @@ -123,11 +126,13 @@ nxt_app_nncq_enqueue(nxt_app_nncq_t volatile *q, nxt_app_nncq_atomic_t val) n = nxt_app_nncq_new_entry(q, t_cycle, val); if (nxt_atomic_cmp_set(&q->entries[j], e, n)) { - break; + nxt_app_nncq_tail_cmp_inc(q, t); + + return NXT_OK; } } - nxt_app_nncq_tail_cmp_inc(q, t); + return NXT_ERROR; } @@ -135,9 +140,9 @@ static nxt_app_nncq_atomic_t nxt_app_nncq_dequeue(nxt_app_nncq_t volatile *q) { nxt_app_nncq_cycle_t e_cycle, h_cycle; - nxt_app_nncq_atomic_t h, j, e; + nxt_app_nncq_atomic_t h, j, e, r; - for ( ;; ) { + for (r = 0; r < NXT_APP_NNCQ_RETRIES; r++) { h = nxt_app_nncq_head(q); j = nxt_app_nncq_map(q, h); e = q->entries[j]; @@ -154,11 +159,11 @@ nxt_app_nncq_dequeue(nxt_app_nncq_t volatile *q) } if (nxt_atomic_cmp_set(&q->head, h, h + 1)) { - break; + return nxt_app_nncq_index(q, e); } } - return nxt_app_nncq_index(q, e); + return nxt_app_nncq_empty(q); } diff --git a/src/nxt_app_queue.h b/src/nxt_app_queue.h index 747611d45..4364bf35a 100644 --- a/src/nxt_app_queue.h +++ b/src/nxt_app_queue.h @@ -75,7 +75,11 @@ nxt_app_queue_send(nxt_app_queue_t volatile *q, const void *p, qi->tracking = tracking; *cookie = i; - nxt_app_nncq_enqueue(&q->queue, i); + if (nxt_slow_path(nxt_app_nncq_enqueue(&q->queue, i) != NXT_OK)) { + (void) nxt_app_nncq_enqueue(&q->free_items, i); + + return NXT_ERROR; + } n = nxt_atomic_cmp_set(&q->notified, 0, 1); @@ -138,7 +142,7 @@ nxt_app_queue_recv(nxt_app_queue_t volatile *q, void *p, uint32_t *cookie) nxt_memcpy(p, qi->data, size); *cookie = i; - nxt_app_nncq_enqueue(&q->free_items, i); + (void) nxt_app_nncq_enqueue(&q->free_items, i); return size; } diff --git a/src/nxt_nncq.h b/src/nxt_nncq.h index 6c9ab3262..28585b227 100644 --- a/src/nxt_nncq.h +++ b/src/nxt_nncq.h @@ -11,6 +11,14 @@ #define NXT_NNCQ_SIZE 16384 +/* + * The peer maps the queue writable and can put a foreign cycle under head + * or tail. Each loop below retries at most this many times, so the peer + * cannot make this side spin for ever. Legitimate contention needs far + * fewer: each retry means another party made progress. + */ +#define NXT_NNCQ_RETRIES NXT_NNCQ_SIZE + typedef uint32_t nxt_nncq_atomic_t; typedef uint16_t nxt_nncq_cycle_t; @@ -94,13 +102,13 @@ nxt_nncq_init(nxt_nncq_t volatile *q) } -static inline void +static inline nxt_int_t nxt_nncq_enqueue(nxt_nncq_t volatile *q, nxt_nncq_atomic_t val) { nxt_nncq_cycle_t e_cycle, t_cycle; - nxt_nncq_atomic_t n, t, e, j; + nxt_nncq_atomic_t n, t, e, j, r; - for ( ;; ) { + for (r = 0; r < NXT_NNCQ_RETRIES; r++) { t = nxt_nncq_tail(q); j = nxt_nncq_map(q, t); e = q->entries[j]; @@ -120,11 +128,13 @@ nxt_nncq_enqueue(nxt_nncq_t volatile *q, nxt_nncq_atomic_t val) n = nxt_nncq_new_entry(q, t_cycle, val); if (nxt_atomic_cmp_set(&q->entries[j], e, n)) { - break; + nxt_nncq_tail_cmp_inc(q, t); + + return NXT_OK; } } - nxt_nncq_tail_cmp_inc(q, t); + return NXT_ERROR; } @@ -132,9 +142,9 @@ static inline nxt_nncq_atomic_t nxt_nncq_dequeue(nxt_nncq_t volatile *q) { nxt_nncq_cycle_t e_cycle, h_cycle; - nxt_nncq_atomic_t h, j, e; + nxt_nncq_atomic_t h, j, e, r; - for ( ;; ) { + for (r = 0; r < NXT_NNCQ_RETRIES; r++) { h = nxt_nncq_head(q); j = nxt_nncq_map(q, h); e = q->entries[j]; @@ -151,11 +161,11 @@ nxt_nncq_dequeue(nxt_nncq_t volatile *q) } if (nxt_atomic_cmp_set(&q->head, h, h + 1)) { - break; + return nxt_nncq_index(q, e); } } - return nxt_nncq_index(q, e); + return nxt_nncq_empty(q); } diff --git a/src/nxt_port_queue.h b/src/nxt_port_queue.h index 50151fb64..ff6d7be7e 100644 --- a/src/nxt_port_queue.h +++ b/src/nxt_port_queue.h @@ -74,7 +74,12 @@ nxt_port_queue_send(nxt_port_queue_t volatile *q, const void *p, uint8_t size, qi->size = size; nxt_memcpy(qi->data, p, size); - nxt_nncq_enqueue(&q->queue, i); + if (nxt_slow_path(nxt_nncq_enqueue(&q->queue, i) != NXT_OK)) { + (void) nxt_nncq_enqueue(&q->free_items, i); + + *notify = 0; + return NXT_ERROR; + } i = nxt_atomic_fetch_add(&q->nitems, 1); @@ -114,7 +119,7 @@ nxt_port_queue_recv(nxt_port_queue_t volatile *q, void *p) nxt_memcpy(p, qi->data, size); - nxt_nncq_enqueue(&q->free_items, i); + (void) nxt_nncq_enqueue(&q->free_items, i); nxt_atomic_fetch_add(&q->nitems, -1); diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 53d8b6609..73bc66347 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -343,6 +343,10 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, (int) port->pid, (int) port->id, port->socket.fd, (int) qmsg_size, notify, res); + if (nxt_slow_path(res == NXT_ERROR)) { + goto queue_broken; + } + if (b != NULL && nxt_fast_path(res == NXT_OK)) { if (qmsg.pm.mmap) { b->is_port_mmap_sent = 1; @@ -377,6 +381,10 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, (int) port->pid, (int) port->id, port->socket.fd, notify, res); + if (nxt_slow_path(res == NXT_ERROR)) { + goto queue_broken; + } + if (nxt_slow_path(res == NXT_AGAIN)) { return NXT_AGAIN; } @@ -444,6 +452,23 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, } return res; + +queue_broken: + + /* + * The peer broke the shared queue and nxt_nncq.h gave up on it. + * Nothing of the message was consumed, so the caller still owns fd, + * fd2 and b, as after a send to a dead peer. Later messages meet the + * same bounded retries: the queue cannot be dropped from a port that + * other engines send on. + */ + + nxt_alert(task, "port{%d,%d} %d: shared queue is broken; " + "message type %d stream #%uD refused", + (int) port->pid, (int) port->id, port->socket.fd, + (int) msg.port_msg.type, stream); + + return NXT_ERROR; } diff --git a/src/test/nxt_nncq_bound_test.c b/src/test/nxt_nncq_bound_test.c new file mode 100644 index 000000000..d187d6206 --- /dev/null +++ b/src/test/nxt_nncq_bound_test.c @@ -0,0 +1,182 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * The application maps the shared memory queues (src/nxt_nncq.h, + * src/nxt_app_nncq.h) writable. A foreign cycle under head or tail used + * to keep the router retrying for ever. Each case poisons one queue and + * runs the router-side operation in a child under alarm(): it must return + * and report the queue empty or the send failed. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static const uint8_t nxt_nncq_bound_test_msg[1] = { 0x5A }; + + +static int +nxt_nncq_bound_test_port_recv(void *mem) +{ + uint8_t buf[NXT_PORT_QUEUE_MSG_SIZE]; + nxt_port_queue_t *q; + + q = mem; + q->queue.entries[0] = 7 * NXT_NNCQ_SIZE; + + return nxt_port_queue_recv(q, buf) == -1; +} + + +static int +nxt_nncq_bound_test_port_send(nxt_port_queue_t *q, nxt_nncq_t *poison) +{ + int notify; + + poison->entries[0] = 7 * NXT_NNCQ_SIZE; + + return nxt_port_queue_send(q, nxt_nncq_bound_test_msg, 1, ¬ify); +} + + +static int +nxt_nncq_bound_test_port_send_full(void *mem) +{ + nxt_port_queue_t *q = mem; + + return nxt_nncq_bound_test_port_send(q, &q->free_items) == NXT_AGAIN; +} + + +/* The refused message must give its slot back: all of them stay free. */ + +static int +nxt_nncq_bound_test_port_send_slot(void *mem) +{ + nxt_uint_t n; + nxt_port_queue_t *q = mem; + + if (nxt_nncq_bound_test_port_send(q, &q->queue) != NXT_ERROR) { + return 0; + } + + for (n = 0; nxt_nncq_dequeue(&q->free_items) + != nxt_nncq_empty(&q->free_items); n++); + + return n == NXT_PORT_QUEUE_SIZE; +} + + +static int +nxt_nncq_bound_test_app_send(nxt_app_queue_t *q, nxt_app_nncq_t *poison) +{ + int notify; + uint32_t cookie; + + poison->entries[0] = 7 * NXT_APP_NNCQ_SIZE; + + return nxt_app_queue_send(q, nxt_nncq_bound_test_msg, 1, 1, ¬ify, + &cookie); +} + + +static int +nxt_nncq_bound_test_app_send_full(void *mem) +{ + nxt_app_queue_t *q = mem; + + return nxt_nncq_bound_test_app_send(q, &q->free_items) == NXT_AGAIN; +} + + +static int +nxt_nncq_bound_test_app_send_slot(void *mem) +{ + nxt_uint_t n; + nxt_app_queue_t *q = mem; + + if (nxt_nncq_bound_test_app_send(q, &q->queue) != NXT_ERROR) { + return 0; + } + + for (n = 0; nxt_app_nncq_dequeue(&q->free_items) + != nxt_app_nncq_empty(&q->free_items); n++); + + return n == NXT_APP_QUEUE_SIZE; +} + + +typedef struct { + const char *name; + nxt_bool_t app; + int (*op)(void *q); +} nxt_nncq_bound_test_case_t; + + +static const nxt_nncq_bound_test_case_t nxt_nncq_bound_test_cases[] = { + { "port recv", 0, nxt_nncq_bound_test_port_recv }, + { "port send, poisoned free list", 0, nxt_nncq_bound_test_port_send_full }, + { "port send, poisoned slot", 0, nxt_nncq_bound_test_port_send_slot }, + { "app send, poisoned free list", 1, nxt_nncq_bound_test_app_send_full }, + { "app send, poisoned slot", 1, nxt_nncq_bound_test_app_send_slot }, +}; + + +static void *nxt_nncq_bound_test_mem; + + +static int +nxt_nncq_bound_test_child(void *data) +{ + const nxt_nncq_bound_test_case_t *tc = data; + + alarm(5); + + return !tc->op(nxt_nncq_bound_test_mem); +} + + +nxt_int_t +nxt_nncq_bound_test(nxt_thread_t *thr) +{ + int rc; + size_t size; + nxt_uint_t i; + + const nxt_nncq_bound_test_case_t *tc; + + nxt_thread_time_update(thr); + + for (i = 0; i < nxt_nitems(nxt_nncq_bound_test_cases); i++) { + tc = &nxt_nncq_bound_test_cases[i]; + size = tc->app ? sizeof(nxt_app_queue_t) : sizeof(nxt_port_queue_t); + + nxt_nncq_bound_test_mem = nxt_malloc(size); + if (nxt_nncq_bound_test_mem == NULL) { + return NXT_ERROR; + } + + if (tc->app) { + nxt_app_queue_init(nxt_nncq_bound_test_mem); + + } else { + nxt_port_queue_init(nxt_nncq_bound_test_mem); + } + + rc = nxt_test_in_child(thr, tc->name, nxt_nncq_bound_test_child, + (void *) tc); + + nxt_free(nxt_nncq_bound_test_mem); + + NXT_TEST_CHECK(thr->log, rc == 0, "nncq bound test: %s did not " + "return the failure", tc->name); + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nncq bound test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 4eba1b5eb..862c4ec67 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -73,7 +73,7 @@ nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), static nxt_int_t (*const nxt_security_tests[])(nxt_thread_t *) = { nxt_checked_test, nxt_port_mmap_read_test, nxt_router_response_parse_test, nxt_port_frag_test, - nxt_port_release_test, + nxt_port_release_test, nxt_nncq_bound_test, }; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index ce63a26a7..7f1b42cce 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -104,6 +104,7 @@ nxt_int_t nxt_conn_close_test(nxt_thread_t *thr); nxt_int_t nxt_router_response_parse_test(nxt_thread_t *thr); nxt_int_t nxt_port_frag_test(nxt_thread_t *thr); nxt_int_t nxt_port_release_test(nxt_thread_t *thr); +nxt_int_t nxt_nncq_bound_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); From c05a45dcea96b0a4694a150ba21cb248bb7c1f94 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 01:39:11 +0200 Subject: [PATCH 301/391] docs: document the C test suite and --hardening in CONTRIBUTING.md Contributors who touch core C or libunit had no instructions for building and running the C tests, only the Python suite. Point them at ./configure --tests, make tests, and the new hook gate script from PR #413 (96f88f76), and say what --tests does and does not change. Also tell contributors to build with --hardening=strict before submitting, since new code must be warning-free under it, and to use the nxt_checked.h and nxt_span.h helpers added in PR #444 (1e73d92d) for arithmetic and parsing on untrusted input. Co-Authored-By: Claude Fable 5.1 --- CONTRIBUTING.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61383d9b1..20a82bf7a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,6 +65,35 @@ Run the test suite before submitting: $ sudo pytest-3 --print-log test/ ``` +If you changed core C code or libunit, also run the C test suite: + +```console +$ ./configure --tests --openssl +$ make +$ make build/lib/libunit.a +$ make tests +$ ./build/tests +``` + +`--tests` builds the test programs (`build/tests`, +`build/unit_port_recv_test`, and the rest). It does not change `unitd`, +`libnxt.a`, `libunit.a`, or any language module: those come from the +same objects with or without `--tests`. + +If you touch a source file under `#if (NXT_TESTS)`, run the gate script +after `make tests`. It fails if a test hook reaches a shipped artifact: + +```console +$ .github/scripts/check-test-hooks.sh +``` + +Build once with `./configure --hardening=strict` before you submit. +New code must compile warning-free under it: no variable-length +arrays, and `nxt_fallthrough;` instead of a fall-through comment. For +length arithmetic on untrusted input, use `nxt_size_add()` and +`nxt_size_mul()` from `src/nxt_checked.h`. For record parsing, use +`nxt_span_t` from `src/nxt_span.h`. + ## Commit Messages Use conventional commits format: From af66a8785ab8f3eeca25d0b847a0d9bcc294eb77 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 16:43:42 +0200 Subject: [PATCH 302/391] router: align the declaration block of nxt_router_app_prepare_request() Co-Authored-By: Claude Fable 5.1 --- src/nxt_router.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index bee06522d..69358143f 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -7589,10 +7589,10 @@ static void nxt_router_app_prepare_request(nxt_task_t *task, nxt_request_rpc_data_t *req_rpc_data) { - nxt_app_t *app; - nxt_buf_t *buf, *body; - nxt_int_t res; - nxt_port_t *port, *reply_port; + nxt_app_t *app; + nxt_buf_t *buf, *body; + nxt_int_t res; + nxt_port_t *port, *reply_port; nxt_http_status_t status; int notify; From d92723b3953dcf6f06f84a39cd3e8b5e48e5c60b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 16:43:42 +0200 Subject: [PATCH 303/391] router: leave skipped fields out of the app message size and the name check nxt_router_prepare_msg() counts every field when it sizes the message, but the copy loop leaves out the fields marked skip. The count only oversized the buffer, but the name-length check added in 593f9f5c ran on skipped fields too, so a field that would never be copied could refuse the request with 431. Today the only request field skipped ahead of this point is Transfer-Encoding (nxt_h1p_transfer_encoding() marks it at parse time, nxt_http_request_chunked_transform() again), so the 431 half is unreachable and a chunked request's message was only sized a field too large. The sizing loop now skips the same fields as the copy loop, which also keeps a future skip from being wrong. The C test builds a second field marked skip with a 251- or 255-byte name next to a live one. The old code refused it with 431; now it is accepted, the message holds one field, and its size is the same as without the skipped field. Co-Authored-By: Claude Fable 5.1 --- src/nxt_router.c | 4 ++ src/test/nxt_router_prepare_msg_test.c | 96 ++++++++++++++++++++------ 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 69358143f..1a421a00b 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -7766,6 +7766,10 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, nxt_http_fields_each(field, r->inline_fields, r->num_inline_fields, r->fields) { + if (field->skip) { + continue; + } + fields_count++; if (nxt_slow_path(field->name_length + prefix->length > UINT8_MAX)) { diff --git a/src/test/nxt_router_prepare_msg_test.c b/src/test/nxt_router_prepare_msg_test.c index 8939ce1c4..628206e94 100644 --- a/src/test/nxt_router_prepare_msg_test.c +++ b/src/test/nxt_router_prepare_msg_test.c @@ -9,8 +9,10 @@ * or a method over 255 bytes, used to wrap and be stored truncated; now they * are refused with 431 and 501. The limits themselves are pinned. * - * The request is built by hand with one field; the engine is a stub that - * only provides the memory pool the buffer is allocated from. The cases + * The request is built by hand with one field, and in some cases a second + * one marked skip, which the router must leave out of the message and its + * size; the engine is a stub that only provides the memory pool the buffer + * is allocated from. The cases * run in a child process: the accepted cases' buffers hold shared memory * that only a real engine can release, and the child's exit frees it. */ @@ -38,31 +40,36 @@ typedef struct { nxt_bool_t prefix; size_t method_length; size_t field_length; - nxt_http_status_t status; /* 0: accepted */ + size_t skipped_length; /* 0: no skipped field */ + nxt_http_status_t status; /* 0: accepted */ } nxt_prepare_msg_test_case_t; static const nxt_prepare_msg_test_case_t nxt_prepare_msg_test_cases[] = { - { "short name, prefix", 1, 3, 6, 0 }, - { "250-byte name + prefix = 255", 1, 3, 250, 0 }, - { "251-byte name + prefix", 1, 3, 251, + { "short name, prefix", 1, 3, 6, 0, 0 }, + { "250-byte name + prefix = 255", 1, 3, 250, 0, 0 }, + { "251-byte name + prefix", 1, 3, 251, 0, NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, - { "253-byte name + prefix", 1, 3, 253, + { "253-byte name + prefix", 1, 3, 253, 0, NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, - { "255-byte name + prefix", 1, 3, 255, + { "255-byte name + prefix", 1, 3, 255, 0, NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, - { "255-byte name, no prefix", 0, 3, 255, 0 }, - { "255-byte method", 0, 255, 6, 0 }, - { "256-byte method", 0, 256, 6, + { "255-byte name, no prefix", 0, 3, 255, 0, 0 }, + { "255-byte method", 0, 255, 6, 0, 0 }, + { "256-byte method", 0, 256, 6, 0, NXT_HTTP_NOT_IMPLEMENTED }, - { "280-byte method", 0, 280, 6, + { "280-byte method", 0, 280, 6, 0, NXT_HTTP_NOT_IMPLEMENTED }, + { "251-byte skipped name + prefix", 1, 3, 6, 251, 0 }, + { "255-byte skipped name + prefix", 1, 3, 6, 255, 0 }, }; +/* On success *used is the size of the built message, 0 when refused. */ + static nxt_int_t nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, - nxt_sockaddr_t *sa, const nxt_prepare_msg_test_case_t *tc) + nxt_sockaddr_t *sa, const nxt_prepare_msg_test_case_t *tc, size_t *used) { size_t expect; nxt_buf_t *b; @@ -103,6 +110,19 @@ nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, field->value = (u_char *) "v"; field->value_length = 1; + if (tc->skipped_length != 0) { + field = &r->inline_fields[1]; + r->num_inline_fields = 2; + + field->hash = 0; + field->skip = 1; + field->name = nxt_prepare_msg_test_name; + field->name_length = tc->skipped_length; + field->value = (u_char *) "v"; + field->value_length = 1; + } + + *used = 0; status = 0; b = nxt_router_test_prepare_msg(task, r, app, tc->prefix, &status); @@ -118,6 +138,8 @@ nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, return NXT_OK; } + *used = nxt_buf_mem_used_size(&b->mem); + req = (nxt_unit_request_t *) b->mem.pos; f = &req->fields[0]; @@ -144,14 +166,17 @@ nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, static nxt_int_t nxt_prepare_msg_test_run(nxt_thread_t *thr) { - nxt_mp_t *mp; - nxt_str_t addr; - nxt_int_t ret; - nxt_app_t *app; - nxt_uint_t i; - nxt_task_t *task; - nxt_sockaddr_t *sa; - nxt_event_engine_t engine, *saved_engine; + size_t used, live_used; + nxt_mp_t *mp; + nxt_str_t addr; + nxt_int_t ret; + nxt_app_t *app; + nxt_uint_t i; + nxt_task_t *task; + nxt_sockaddr_t *sa; + nxt_event_engine_t engine, *saved_engine; + nxt_prepare_msg_test_case_t live; + const nxt_prepare_msg_test_case_t *tc; nxt_thread_time_update(thr); @@ -191,11 +216,36 @@ nxt_prepare_msg_test_run(nxt_thread_t *thr) ret = NXT_OK; for (i = 0; i < nxt_nitems(nxt_prepare_msg_test_cases); i++) { - if (nxt_prepare_msg_test_case(task, mp, app, sa, - &nxt_prepare_msg_test_cases[i]) + tc = &nxt_prepare_msg_test_cases[i]; + + if (nxt_prepare_msg_test_case(task, mp, app, sa, tc, &used) + != NXT_OK) + { + ret = NXT_ERROR; + continue; + } + + if (tc->skipped_length == 0 || tc->status != 0) { + continue; + } + + /* The skipped field must add nothing to the message. */ + + live = *tc; + live.skipped_length = 0; + + if (nxt_prepare_msg_test_case(task, mp, app, sa, &live, &live_used) != NXT_OK) { ret = NXT_ERROR; + continue; + } + + if (used != live_used) { + nxt_log_alert(task->log, "prepare msg test \"%s\": the message " + "is %uz bytes, %uz without the skipped field", + tc->name, used, live_used); + ret = NXT_ERROR; } } From 824d1bec6f9ef61442e27563ccfccde874ec55d9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 16:43:42 +0200 Subject: [PATCH 304/391] router: log a failed app message once Every branch of nxt_router_prepare_msg() that fails with 500 logs its reason, and the shared memory path alerts on each failure but one: the mmap and shm_open failures name the call that failed, but nxt_port_mmap_get_buf() said nothing when the buffer header allocation failed. So the caller's alert, "failed to prepare app message", repeated every event except that one. Log that one in nxt_port_mmap_get_buf(), where the certificate store and the websocket path get it as well, and drop the caller's alert. Name the application in the four reason lines of nxt_router_prepare_msg() that did not, since that is what the dropped line added. Co-Authored-By: Claude Fable 5.1 --- src/nxt_port_memory.c | 3 +++ src/nxt_router.c | 25 +++++++++++-------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 2fa843efd..e680013c1 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -713,6 +713,9 @@ nxt_port_mmap_get_buf(nxt_task_t *task, nxt_port_mmaps_t *mmaps, size_t size) b = nxt_buf_mem_ts_alloc(task, task->thread->engine->mem_pool, 0); if (nxt_slow_path(b == NULL)) { + nxt_alert(task, "failed to allocate a buffer for %z bytes of shared " + "memory", size); + return NULL; } diff --git a/src/nxt_router.c b/src/nxt_router.c index 1a421a00b..5fd09bc32 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -7616,11 +7616,6 @@ nxt_router_app_prepare_request(nxt_task_t *task, buf = nxt_router_prepare_msg(task, req_rpc_data->request, app, nxt_app_msg_prefix[app->type], &status); if (nxt_slow_path(buf == NULL)) { - if (status == NXT_HTTP_INTERNAL_SERVER_ERROR) { - nxt_alert(task, "stream #%uD, app '%V': failed to prepare app " - "message", req_rpc_data->stream, &app->name); - } - nxt_http_request_error(task, req_rpc_data->request, status); return; @@ -7730,8 +7725,9 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, *status = NXT_HTTP_INTERNAL_SERVER_ERROR; if (nxt_slow_path(r->method->length > UINT8_MAX)) { - nxt_log(task, NXT_LOG_INFO, "request method of %uz bytes is too long " - "for the application protocol", r->method->length); + nxt_log(task, NXT_LOG_INFO, "app '%V': request method of %uz bytes " + "is too long for the application protocol", &app->name, + r->method->length); *status = NXT_HTTP_NOT_IMPLEMENTED; return NULL; @@ -7742,8 +7738,8 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, || r->local->address_length > UINT8_MAX || nxt_sockaddr_port_length(r->local) > UINT8_MAX)) { - nxt_alert(task, "request version or address too long for the " - "application protocol"); + nxt_alert(task, "app '%V': request version or address too long for " + "the application protocol", &app->name); return NULL; } @@ -7773,9 +7769,10 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, fields_count++; if (nxt_slow_path(field->name_length + prefix->length > UINT8_MAX)) { - nxt_log(task, NXT_LOG_INFO, "header field name of %d bytes is " - "too long for the application protocol with prefix " - "\"%V\"", (int) field->name_length, prefix); + nxt_log(task, NXT_LOG_INFO, "app '%V': header field name of %d " + "bytes is too long for the application protocol with " + "prefix \"%V\"", &app->name, (int) field->name_length, + prefix); *status = NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE; return NULL; @@ -7788,8 +7785,8 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, req_size += fields_count * sizeof(nxt_unit_field_t); if (nxt_slow_path(req_size > PORT_MMAP_DATA_SIZE)) { - nxt_alert(task, "headers too big to fit in shared memory (%uz)", - req_size); + nxt_alert(task, "app '%V': headers too big to fit in shared memory " + "(%uz)", &app->name, req_size); return NULL; } From 0bdeb827de12409ec374f62b768484bf78852123 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 16:44:37 +0200 Subject: [PATCH 305/391] tests: pin nxt_app_msg_prefix[] in the prepare-msg test, and declare its hook once The test hook took a boolean and chose between the two prefixes itself, so the C test could not tell which app types the router prefixes; the end-to-end test covers PHP only, and Perl and Ruby share the prefix while five types have none. The hook now takes the prefix from nxt_app_msg_prefix[app->type], and the cases name the app type: 251-byte names are refused for PHP, Perl and Ruby, 255-byte names are accepted for the rest, and an accepted name starts with "HTTP_" exactly when the type is prefixed. A flipped table entry now fails ./build/tests. The hook's status is an nxt_uint_t, so the prototype lives in the NXT_TESTS block of nxt_router.h with the other hooks instead of being repeated in the test. Co-Authored-By: Claude Fable 5.1 --- src/nxt_router.c | 23 +++----- src/nxt_router.h | 8 +++ src/test/nxt_router_prepare_msg_test.c | 79 +++++++++++++++++--------- 3 files changed, 69 insertions(+), 41 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 5fd09bc32..773698066 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -8032,23 +8032,18 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, #if (NXT_TESTS) -/* - * For src/test/nxt_router_prepare_msg_test.c, which repeats this prototype: - * nxt_router.h cannot name nxt_http_status_t. - */ - -nxt_buf_t *nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, - nxt_app_t *app, nxt_bool_t use_http_prefix, nxt_http_status_t *status); - - nxt_buf_t * nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, - nxt_app_t *app, nxt_bool_t use_http_prefix, nxt_http_status_t *status) + nxt_app_t *app, nxt_uint_t *status) { - return nxt_router_prepare_msg(task, r, app, - use_http_prefix ? &http_prefix - : &empty_prefix, - status); + nxt_buf_t *b; + nxt_http_status_t st; + + b = nxt_router_prepare_msg(task, r, app, nxt_app_msg_prefix[app->type], + &st); + *status = st; + + return b; } #endif diff --git a/src/nxt_router.h b/src/nxt_router.h index 56b7f7014..c593a8ade 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -387,6 +387,14 @@ void nxt_router_test_app_timeout(nxt_task_t *task, void *obj, void *data); /* The application response decoder, for the response parse test. */ nxt_int_t nxt_router_test_response_header_parse(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *b); + +/* + * The app message builder with the prefix of app->type, for the protocol + * length test. The status is an nxt_uint_t: nxt_http_status_t is not + * visible here. + */ +nxt_buf_t *nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, + nxt_app_t *app, nxt_uint_t *status); #endif diff --git a/src/test/nxt_router_prepare_msg_test.c b/src/test/nxt_router_prepare_msg_test.c index 628206e94..02a0394f9 100644 --- a/src/test/nxt_router_prepare_msg_test.c +++ b/src/test/nxt_router_prepare_msg_test.c @@ -7,7 +7,8 @@ * nxt_router_prepare_msg() (src/nxt_router.c) stores the method and header * name lengths as uint8_t. A 251..255-byte name plus the "HTTP_" prefix, * or a method over 255 bytes, used to wrap and be stored truncated; now they - * are refused with 431 and 501. The limits themselves are pinned. + * are refused with 431 and 501. The limits themselves are pinned, and so + * is nxt_app_msg_prefix[]: which app types get the prefix and which do not. * * The request is built by hand with one field, and in some cases a second * one marked skip, which the router must leave out of the message and its @@ -27,17 +28,13 @@ #include "nxt_tests.h" -nxt_buf_t *nxt_router_test_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, - nxt_app_t *app, nxt_bool_t use_http_prefix, nxt_http_status_t *status); - - static u_char nxt_prepare_msg_test_name[300]; static u_char nxt_prepare_msg_test_method[300]; typedef struct { const char *name; - nxt_bool_t prefix; + nxt_app_type_t type; size_t method_length; size_t field_length; size_t skipped_length; /* 0: no skipped field */ @@ -45,38 +42,62 @@ typedef struct { } nxt_prepare_msg_test_case_t; +#define NXT_TEST_431 NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE +#define NXT_TEST_501 NXT_HTTP_NOT_IMPLEMENTED + + static const nxt_prepare_msg_test_case_t nxt_prepare_msg_test_cases[] = { - { "short name, prefix", 1, 3, 6, 0, 0 }, - { "250-byte name + prefix = 255", 1, 3, 250, 0, 0 }, - { "251-byte name + prefix", 1, 3, 251, 0, - NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, - { "253-byte name + prefix", 1, 3, 253, 0, - NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, - { "255-byte name + prefix", 1, 3, 255, 0, - NXT_HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE }, - { "255-byte name, no prefix", 0, 3, 255, 0, 0 }, - { "255-byte method", 0, 255, 6, 0, 0 }, - { "256-byte method", 0, 256, 6, 0, - NXT_HTTP_NOT_IMPLEMENTED }, - { "280-byte method", 0, 280, 6, 0, - NXT_HTTP_NOT_IMPLEMENTED }, - { "251-byte skipped name + prefix", 1, 3, 6, 251, 0 }, - { "255-byte skipped name + prefix", 1, 3, 6, 255, 0 }, + { "short name, php", NXT_APP_PHP, 3, 6, 0, 0 }, + { "250-byte name + prefix", NXT_APP_PHP, 3, 250, 0, 0 }, + { "251-byte name + prefix", NXT_APP_PHP, 3, 251, 0, + NXT_TEST_431 }, + { "251-byte name, perl", NXT_APP_PERL, 3, 251, 0, + NXT_TEST_431 }, + { "251-byte name, ruby", NXT_APP_RUBY, 3, 251, 0, + NXT_TEST_431 }, + { "253-byte name + prefix", NXT_APP_PHP, 3, 253, 0, + NXT_TEST_431 }, + { "255-byte name + prefix", NXT_APP_PHP, 3, 255, 0, + NXT_TEST_431 }, + { "255-byte name, external", NXT_APP_EXTERNAL, 3, 255, 0, 0 }, + { "255-byte name, python", NXT_APP_PYTHON, 3, 255, 0, 0 }, + { "255-byte name, java", NXT_APP_JAVA, 3, 255, 0, 0 }, + { "255-byte name, wasm", NXT_APP_WASM, 3, 255, 0, 0 }, + { "255-byte name, wasm-wc", NXT_APP_WASM_WC, 3, 255, 0, 0 }, + { "255-byte method", NXT_APP_PYTHON, 255, 6, 0, 0 }, + { "256-byte method", NXT_APP_PYTHON, 256, 6, 0, + NXT_TEST_501 }, + { "280-byte method", NXT_APP_PYTHON, 280, 6, 0, + NXT_TEST_501 }, + { "251-byte skipped name", NXT_APP_PHP, 3, 6, 251, 0 }, + { "255-byte skipped name", NXT_APP_PHP, 3, 6, 255, 0 }, }; +/* The app types whose header names carry "HTTP_"; the rest get none. */ + +static nxt_bool_t +nxt_prepare_msg_test_prefixed(nxt_app_type_t type) +{ + return (type == NXT_APP_PHP || type == NXT_APP_PERL + || type == NXT_APP_RUBY); +} + + /* On success *used is the size of the built message, 0 when refused. */ static nxt_int_t nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, nxt_sockaddr_t *sa, const nxt_prepare_msg_test_case_t *tc, size_t *used) { + u_char *name; size_t expect; nxt_buf_t *b; nxt_str_t *method, *args; + nxt_uint_t status; + nxt_bool_t prefixed; nxt_http_field_t *field; nxt_unit_field_t *f; - nxt_http_status_t status; nxt_unit_request_t *req; nxt_http_request_t *r; @@ -122,13 +143,14 @@ nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, field->value_length = 1; } + app->type = tc->type; *used = 0; status = 0; - b = nxt_router_test_prepare_msg(task, r, app, tc->prefix, &status); + b = nxt_router_test_prepare_msg(task, r, app, &status); if (b == NULL) { - if (status != tc->status) { + if (status != (nxt_uint_t) tc->status) { nxt_log_alert(task->log, "prepare msg test \"%s\": refused " "with %d, expected %d", tc->name, (int) status, (int) tc->status); @@ -142,14 +164,17 @@ nxt_prepare_msg_test_case(nxt_task_t *task, nxt_mp_t *mp, nxt_app_t *app, req = (nxt_unit_request_t *) b->mem.pos; f = &req->fields[0]; + name = nxt_unit_sptr_get(&f->name); - expect = tc->field_length + (tc->prefix ? nxt_length("HTTP_") : 0); + prefixed = nxt_prepare_msg_test_prefixed(tc->type); + expect = tc->field_length + (prefixed ? nxt_length("HTTP_") : 0); if (tc->status != 0 || req->method_length != tc->method_length || req->fields_count != 1 || f->name_length != expect - || nxt_strlen(nxt_unit_sptr_get(&f->name)) != expect) + || nxt_strlen(name) != expect + || (nxt_strncmp(name, "HTTP_", nxt_length("HTTP_")) == 0) != prefixed) { nxt_log_alert(task->log, "prepare msg test \"%s\": accepted with " "method_length %d, name_length %d; expected %d and " From 4a42cbc37de9622431ab9cd91902f6cadfd153c9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 25 Sep 2026 20:29:24 +0000 Subject: [PATCH 306/391] fix(unit): act on a QUIT that arrives on the bare socket The prototype sends a QUIT on the bare socket to a worker that has not sent PROCESS_READY yet. It has not mapped the queue of that worker. No READ_SOCKET mark is queued for such a message. nxt_unit_ctx_port_recv() suspended the QUIT and waited for a mark that never came. The worker ran until the SIGTERM 15 s later, and unitd reported "Unit did not exit within 15s after SIGQUIT". A QUIT needs no order with the queued messages. So return it at once instead of a suspend. nxt_unit_is_socket_quit() accepts the QUIT as nxt_runtime_port_send_quit() writes it: the header, or the header and one byte for the quit mode. The type must be QUIT and the mmap bit must be clear. nxt_unit_is_quit() is unchanged. Add two cases to unit_port_recv_test. They write the QUIT the prototype sends to a context with nothing in its queue. One context is not ready, the other is ready. Without the fix both cases wait until SIGALRM. Fixes #482 Co-Authored-By: Claude Opus 5.5 --- src/nxt_unit.c | 40 +++++++++++++++++ src/test/nxt_unit_port_recv_test.c | 72 ++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 4b7793fb0..a2fde4dbc 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -163,6 +163,7 @@ nxt_inline int nxt_unit_is_read_queue(nxt_unit_read_buf_t *rbuf); nxt_inline int nxt_unit_is_read_socket(nxt_unit_read_buf_t *rbuf); nxt_inline int nxt_unit_is_shm_ack(nxt_unit_read_buf_t *rbuf); nxt_inline int nxt_unit_is_quit(nxt_unit_read_buf_t *rbuf); +nxt_inline int nxt_unit_is_socket_quit(nxt_unit_read_buf_t *rbuf); static int nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); static void nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl); @@ -6046,6 +6047,29 @@ nxt_unit_is_quit(nxt_unit_read_buf_t *rbuf) } +/* + * A QUIT as nxt_runtime_port_send_quit() writes it. It is the header and + * one byte for the quit mode. The byte is absent when the runtime had no + * memory for it. + */ + +nxt_inline int +nxt_unit_is_socket_quit(nxt_unit_read_buf_t *rbuf) +{ + nxt_port_msg_t *port_msg; + + if (rbuf->size == (ssize_t) sizeof(nxt_port_msg_t) + || rbuf->size == (ssize_t) sizeof(nxt_port_msg_t) + 1) + { + port_msg = (nxt_port_msg_t *) rbuf->buf; + + return port_msg->type == _NXT_PORT_MSG_QUIT && !port_msg->mmap; + } + + return 0; +} + + int nxt_unit_run_shared(nxt_unit_ctx_t *ctx) { @@ -7300,6 +7324,22 @@ nxt_unit_ctx_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, return NXT_UNIT_OK; } + /* + * A QUIT with no READ_SOCKET mark ahead of it was not sent through + * the queue, and no mark comes for it later. The prototype writes a + * QUIT to the bare socket of a worker whose queue it has not mapped + * yet. That worker has not sent PROCESS_READY. A suspended QUIT + * keeps the worker alive until SIGTERM. A QUIT needs no order with + * the queued messages, so act on it now. + */ + if (nxt_unit_is_socket_quit(rbuf)) { + nxt_unit_debug(ctx, "port{%d,%d} recv %d quit", + (int) port->id.pid, (int) port->id.id, + (int) rbuf->size); + + return NXT_UNIT_OK; + } + nxt_unit_debug(ctx, "port{%d,%d} suspend message %d", (int) port->id.pid, (int) port->id.id, (int) rbuf->size); diff --git a/src/test/nxt_unit_port_recv_test.c b/src/test/nxt_unit_port_recv_test.c index 5f2021923..1b4e66c4a 100644 --- a/src/test/nxt_unit_port_recv_test.c +++ b/src/test/nxt_unit_port_recv_test.c @@ -57,6 +57,7 @@ static int nxt_port_recv_test_shared_queue_fd = -1; static int nxt_port_recv_test_sock_fd = -1; static int nxt_port_recv_test_queue_fd = -1; static int nxt_port_recv_test_ctx_queue_fd = -1; +static int nxt_port_recv_test_read_out_fd = -1; static void *nxt_port_recv_test_ctx_queue; static struct { @@ -603,6 +604,72 @@ nxt_port_recv_test_deferred_quit_read(nxt_unit_ctx_t *ctx, int run_ctx, } +static void +nxt_port_recv_test_socket_quit(nxt_unit_ctx_t *ctx, uint8_t ready, + const char *name) +{ + pid_t pid; + ssize_t n; + nxt_app_queue_t *q; + + struct { + nxt_port_msg_t msg; + uint8_t quit_param; + } nxt_packed m; + + /* + * The prototype writes a QUIT to the bare socket of a worker that has + * not sent PROCESS_READY. No READ_SOCKET mark is queued for that QUIT. + * The worker must act on it. A suspended QUIT waits for a mark that + * never comes. The child then waits until SIGALRM stops it. + */ + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + /* An empty shared queue, as the router leaves it. */ + q = mmap(NULL, sizeof(nxt_app_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED, nxt_port_recv_test_shared_queue_fd, 0); + if (q == MAP_FAILED) { + _exit(4); + } + + nxt_app_queue_init(q); + + nxt_unit_test_ctx_set_ready(ctx, ready); + nxt_unit_test_ctx_set_detached(ctx, 0); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + /* The message nxt_runtime_port_send_quit() writes. */ + memset(&m, 0, sizeof(m)); + + m.msg.pid = getppid(); + m.msg.type = _NXT_PORT_MSG_QUIT; + m.quit_param = NXT_PORT_QUIT_GRACEFUL; + + n = write(nxt_port_recv_test_read_out_fd, &m, sizeof(m)); + if (n != (ssize_t) sizeof(m)) { + _exit(1); + } + + alarm(5); + + if (nxt_unit_run(ctx) != NXT_UNIT_OK) { + _exit(2); + } + + if (nxt_unit_test_ctx_online(ctx) != 0) { + _exit(3); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + static void nxt_port_recv_test_detached_persistent_fail(nxt_unit_ctx_t *ctx) { @@ -705,6 +772,7 @@ main(void) } nxt_port_recv_test_sock_fd = sock[0]; + nxt_port_recv_test_read_out_fd = read[1]; nxt_port_recv_test_queue_fd = nxt_port_recv_test_shm(sizeof(nxt_port_queue_t)); @@ -821,6 +889,10 @@ main(void) nxt_port_recv_test_detached_idle_retry(ctx, 0, "finish retry reaches the give-up, not ready and idle"); nxt_port_recv_test_detached_persistent_fail(ctx); + nxt_port_recv_test_socket_quit(ctx, 0, + "bare socket quit stops a context not ready"); + nxt_port_recv_test_socket_quit(ctx, 1, + "bare socket quit stops a ready context"); nxt_port_recv_test_deferred_quit_read(ctx, 0, "nxt_unit_run() returns on a retry that completes a quit"); nxt_port_recv_test_deferred_quit_read(ctx, 1, From 5a0621cefe46a0686449d70c3f9f1d3f98c6b0fd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:42:42 +0000 Subject: [PATCH 307/391] fix(router): do not alert when the prototype is gone at worker start On shutdown main sends QUIT to every process directly. This causes a race between the prototype and the router: 1. The prototype gets QUIT and waits for its workers to exit. 2. A worker exits. The prototype reports it to the router (REMOVE_PID). When the last worker is gone, the prototype exits. 3. The router handles this REMOVE_PID before its own QUIT. It starts a replacement worker in nxt_router_app_port_close(). 4. nxt_router_start_app_process_handler() writes START_PROCESS to the prototype socket. Two things can happen at step 4. If the prototype has already exited, sendmsg() fails with EPIPE (or ECONNRESET, ECONNREFUSED), and the router logged [alert] app 'libunit' failed to start a process This alert made test_app_lifecycle_process_churn[libunit] fail at teardown in the sanitize job. A prototype that crashes gives the same result. The router cannot see this in advance. It clears ->proto_port only when it gets the REMOVE_PID of the prototype, or when it sends the QUIT itself. If the prototype is still alive, because another worker has not exited yet, it forked a new worker. Nobody sends QUIT to this new worker. The prototype waits for it, so Unit does not exit. The test suite then fails at teardown with "Unit did not exit within 15s after SIGQUIT". This was seen in test_app_lifecycle.py under ASan. Fix 1, router: when the write of a worker start fails with EPIPE, ECONNRESET or ECONNREFUSED, log "start attempt cancelled: prototype is gone" at debug level instead of the alert. A prototype start goes to main, so it keeps the alert. Every other write error keeps the alert too. Fix 2, application: a prototype that got QUIT refuses START_PROCESS. It closes the descriptors of the message and answers RPC_ERROR, as the other failures in nxt_proto_start_process_handler() do. The router then retires its RPC and gets the slot back. Not changed: the slot accounting was already correct. Nothing is sent, so nothing is forked, and the failure path releases the pending processes slot and the start timer as before. A prototype that crashes is still reported by main. Tests: src/test/nxt_router_start_proto_gone_test.c is new. It sends a worker start to a socket pair with a closed reader end, and checks the slot, the app reference and the alert count. A second case injects EINVAL and checks that the alert is still logged. src/test/nxt_main_start_process_reply_test.c gets a new case. It runs the prototype handler with the exiting flag set, in a forked child, and requires one RPC_ERROR on the stream of the start. Refs #482 for the QUIT that libunit suspends in a new worker. Co-Authored-By: Claude Opus 5.5 --- auto/sources | 1 + src/nxt_application.c | 25 ++ src/nxt_application.h | 1 + src/nxt_router.c | 38 ++- src/test/nxt_main_start_process_reply_test.c | 104 +++++- src/test/nxt_router_start_proto_gone_test.c | 326 +++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 8 files changed, 497 insertions(+), 3 deletions(-) create mode 100644 src/test/nxt_router_start_proto_gone_test.c diff --git a/auto/sources b/auto/sources index e41527f7a..a9163bf7c 100644 --- a/auto/sources +++ b/auto/sources @@ -191,6 +191,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_new_port_test.c \ src/test/nxt_router_start_fail_test.c \ src/test/nxt_router_start_fail_soak_test.c \ + src/test/nxt_router_start_proto_gone_test.c \ src/test/nxt_router_proto_wedge_test.c \ src/test/nxt_router_proto_death_test.c \ src/test/nxt_router_start_timeout_test.c \ diff --git a/src/nxt_application.c b/src/nxt_application.c index d98297ad2..925bb8be8 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -771,6 +771,24 @@ nxt_proto_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } #endif + /* + * A prototype that got QUIT only waits for its workers to exit. The + * router can still send START_PROCESS at that point. On shutdown main + * sends QUIT to the prototype directly. The prototype reports the exit + * of a worker to the router. The router starts a replacement before + * its own QUIT arrives. A worker forked now gets no QUIT from anyone. + * The prototype waits for that worker, so Unit does not exit. Refuse + * the start. The reply fails the RPC of the router and gives the slot + * back. + */ + if (nxt_slow_path(nxt_proto_exiting)) { + nxt_debug(task, "prototype is exiting, start process refused"); + + nxt_port_recv_msg_close_fds(msg); + + goto failed; + } + process = nxt_process_new(rt); if (nxt_slow_path(process == NULL)) { goto failed; @@ -854,6 +872,13 @@ nxt_proto_test_run_start_process_handler(nxt_task_t *task, nxt_proto_start_process_handler(task, msg); } + +void +nxt_proto_test_set_exiting(nxt_bool_t exiting) +{ + nxt_proto_exiting = exiting; +} + #endif diff --git a/src/nxt_application.h b/src/nxt_application.h index 4be7ff42f..36ad6b8d3 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -199,6 +199,7 @@ nxt_app_type_t nxt_app_parse_type(u_char *p, size_t length); */ void nxt_proto_test_run_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +void nxt_proto_test_set_exiting(nxt_bool_t exiting); #endif NXT_EXPORT extern nxt_str_t nxt_server; diff --git a/src/nxt_router.c b/src/nxt_router.c index 773698066..b7f4f9e4e 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -547,9 +547,11 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, size_t size; uint32_t stream; nxt_fd_t port_fd, queue_fd; + nxt_err_t err; nxt_int_t ret; nxt_app_t *app; nxt_buf_t *b; + nxt_bool_t proto_gone; nxt_port_t *dport; nxt_runtime_t *rt; nxt_app_joint_rpc_t *app_joint_rpc; @@ -558,6 +560,7 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, app = data; st = NULL; + proto_gone = 0; nxt_thread_mutex_lock(&app->mutex); @@ -628,6 +631,30 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, if (nxt_slow_path(ret != NXT_OK)) { nxt_port_rpc_cancel(task, port, stream); + /* + * A worker start goes to the prototype. The prototype can exit + * without a request from the router. On shutdown main sends QUIT + * to every process directly. The prototype reports the exit of its + * workers to the router (REMOVE_PID) and then exits. The router + * can handle such a REMOVE_PID before its own QUIT. It then starts + * a replacement worker in nxt_router_app_port_close(), and the + * socket of the prototype is already closed. A prototype that + * crashes gives the same result. The router cannot know this in + * advance: it clears ->proto_port only when the REMOVE_PID of the + * prototype arrives, or when it sends the QUIT itself. + * + * Nothing was sent, so nothing was forked. The slot goes back + * below in the usual way. Only the log level is different. A + * prototype start goes to main. Main is never expected to be gone, + * so that start keeps the alert. + */ + + err = dport->socket.error; + + proto_gone = (b == NULL + && (err == NXT_EPIPE || err == NXT_ECONNRESET + || err == NXT_ECONNREFUSED)); + goto failed; } @@ -680,7 +707,13 @@ nxt_router_start_app_process_handler(nxt_task_t *task, nxt_port_t *port, * nxt_router_app_port_error() does for the attempts that do get that far. */ - nxt_alert(task, "app '%V' failed to start a process", &app->name); + if (proto_gone) { + nxt_debug(task, "app '%V' start attempt cancelled: prototype %PI " + "is gone", &app->name, dport->pid); + + } else { + nxt_alert(task, "app '%V' failed to start a process", &app->name); + } if (st != NULL) { /* @@ -6259,7 +6292,8 @@ nxt_router_app_port_error(nxt_task_t *task, nxt_port_recv_msg_t *msg, * module that fails to import, a worker that exits during startup -- * and Unit is working exactly as designed when it reports one. * [error] is emitted at the default log level, which is the whole - * point of the change. + * point of the change. One exception: a write to a prototype that is + * already gone is logged at debug level there, not as an alert. * * A different sentence from that alert, deliberately, even though * the two describe the same disappointment. The test suite skips diff --git a/src/test/nxt_main_start_process_reply_test.c b/src/test/nxt_main_start_process_reply_test.c index 29c9eabeb..f751f8431 100644 --- a/src/test/nxt_main_start_process_reply_test.c +++ b/src/test/nxt_main_start_process_reply_test.c @@ -64,6 +64,16 @@ #include +typedef void (*nxt_main_start_process_reply_test_handler_t)(nxt_task_t *task, + nxt_port_recv_msg_t *msg); + + +static nxt_int_t nxt_main_start_process_reply_test_answer(nxt_thread_t *thr, + nxt_task_t *task, nxt_port_recv_msg_t *msg, + nxt_main_start_process_reply_test_handler_t handler, + nxt_port_t *reply_port, nxt_port_t *silent_port, const char *name); + + /* * Run the handler once and require that it answered: exactly one RPC_ERROR * on reply_port, carrying the stream of the incoming message, and nothing @@ -75,6 +85,20 @@ static nxt_int_t nxt_main_start_process_reply_test_case(nxt_thread_t *thr, nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_t *reply_port, nxt_port_t *silent_port, const char *name) +{ + return nxt_main_start_process_reply_test_answer(thr, task, msg, + nxt_main_test_run_start_process_handler, + reply_port, silent_port, name); +} + + +/* The same check for any START_PROCESS handler. */ + +static nxt_int_t +nxt_main_start_process_reply_test_answer(nxt_thread_t *thr, nxt_task_t *task, + nxt_port_recv_msg_t *msg, + nxt_main_start_process_reply_test_handler_t handler, + nxt_port_t *reply_port, nxt_port_t *silent_port, const char *name) { nxt_fd_t fd0, fd1; nxt_int_t ret; @@ -98,7 +122,7 @@ nxt_main_start_process_reply_test_case(nxt_thread_t *thr, nxt_task_t *task, msg->fd[0] = fd0; msg->fd[1] = fd1; - nxt_main_test_run_start_process_handler(task, msg); + handler(task, msg); /* * Both exits own the descriptors the message carried, and both spellings @@ -398,6 +422,74 @@ nxt_main_start_process_reply_test_proto_case(nxt_thread_t *thr, #endif +/* + * A START_PROCESS from the router reaches a prototype after its QUIT. On + * shutdown this happens when the exiting prototype reports the exit of a + * worker, and the router starts a replacement for that worker. A worker + * forked at that point gets no QUIT from anyone, and the prototype waits for + * it for ever. The prototype must refuse the start. It must answer, so + * that the router retires its RPC. + * + * The case runs in a forked child, like the refusal cases above. A handler + * that does not refuse forks a worker from a fixture that has no application + * loaded. + */ +static nxt_int_t +nxt_main_start_process_reply_test_proto_exiting(nxt_thread_t *thr, + nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_port_t *reply_port, + nxt_port_t *silent_port) +{ + int status; + pid_t child; + nxt_int_t ret; + + child = fork(); + + if (nxt_slow_path(child == -1)) { + nxt_log_alert(thr->log, "main start process reply test: an exiting " + "prototype failed to fork %E", nxt_errno); + return NXT_ERROR; + } + + if (child == 0) { + nxt_proto_test_set_exiting(1); + + ret = nxt_main_start_process_reply_test_answer(thr, task, msg, + nxt_proto_test_run_start_process_handler, + reply_port, silent_port, + "an exiting prototype"); + + _exit(ret == NXT_OK ? 0 : 1); + } + + while (waitpid(child, &status, 0) == -1) { + if (nxt_errno != NXT_EINTR) { + nxt_log_alert(thr->log, "main start process reply test: an " + "exiting prototype failed to reap the child %E", + nxt_errno); + return NXT_ERROR; + } + } + + if (nxt_slow_path(!WIFEXITED(status))) { + nxt_log_alert(thr->log, "main start process reply test: an exiting " + "prototype did not refuse -- the handler went on to " + "start a worker, and died there on signal %d", + WTERMSIG(status)); + return NXT_ERROR; + } + + if (nxt_slow_path(WEXITSTATUS(status) != 0)) { + nxt_log_alert(thr->log, "main start process reply test: an exiting " + "prototype failed, exit status %d", + WEXITSTATUS(status)); + return NXT_ERROR; + } + + return NXT_OK; +} + + nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr) { @@ -692,6 +784,16 @@ nxt_main_start_process_reply_test(nxt_thread_t *thr) #endif + msg.port_msg.stream = 0x88888888; + + ret = nxt_main_start_process_reply_test_proto_exiting(thr, task, &msg, + router_port, + foreign_port); + + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + nxt_thread_time_update(thr); nxt_log_error(NXT_LOG_NOTICE, thr->log, "main start process reply test passed"); diff --git a/src/test/nxt_router_start_proto_gone_test.c b/src/test/nxt_router_start_proto_gone_test.c new file mode 100644 index 000000000..cf8d153bd --- /dev/null +++ b/src/test/nxt_router_start_proto_gone_test.c @@ -0,0 +1,326 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test: a worker start sent to a prototype that has already + * exited must not be logged as an alert. + * + * On shutdown main sends QUIT to every process directly. The prototype + * reports the exit of its workers to the router (REMOVE_PID) and then exits. + * The router can handle such a REMOVE_PID before its own QUIT. It then + * starts a replacement worker from nxt_router_app_port_close(), and writes + * START_PROCESS to the socket of the prototype. The prototype has closed + * that socket. sendmsg() fails with EPIPE (ECONNREFUSED on a SOCK_DGRAM + * pair), and nxt_router_start_app_process_handler() logged + * "app '...' failed to start a process" as an alert. The test suite fails + * on that alert (test_app_lifecycle_process_churn in the sanitize job). + * + * The first case sends to a real socket pair. The reader end of the pair is + * closed. The case checks that the slot is released and that the alert is + * not logged. The second case makes the send fail with an error that does + * not mean "the peer is gone". The case checks that the alert is logged. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +static void nxt_cdecl nxt_router_start_proto_gone_test_log_handler( + nxt_uint_t level, nxt_log_t *log, const char *fmt, ...); +static nxt_int_t nxt_router_start_proto_gone_test_run(nxt_thread_t *thr, + nxt_err_t inject, nxt_uint_t expected_alerts); +static void nxt_router_start_proto_gone_test_drain_wq(nxt_work_queue_t *wq); + + +/* + * The handler passes the application to the reference-counting helpers of + * the router. The storage must stay valid after the test function returns, + * so the application has file scope. + */ +static nxt_app_t nxt_router_start_proto_gone_test_app; + +static nxt_uint_t nxt_router_start_proto_gone_test_alerts; + + +nxt_int_t +nxt_router_start_proto_gone_test(nxt_thread_t *thr) +{ + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test started"); + + /* The prototype has closed its socket: no alert. */ + + if (nxt_router_start_proto_gone_test_run(thr, 0, 0) != NXT_OK) { + return NXT_ERROR; + } + + /* Any other send failure is still an alert. */ + + if (nxt_router_start_proto_gone_test_run(thr, NXT_EINVAL, 1) != NXT_OK) { + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test passed"); + + return NXT_OK; +} + + +/* + * Count the "failed to start a process" alerts. Pass every line on to the + * main log, so that the test output stays readable. + */ + +static void nxt_cdecl +nxt_router_start_proto_gone_test_log_handler(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...) +{ + u_char *p; + va_list args; + u_char msg[NXT_MAX_ERROR_STR]; + + va_start(args, fmt); + p = nxt_vsprintf(msg, msg + NXT_MAX_ERROR_STR - 1, fmt, args); + va_end(args); + + *p = '\0'; + + if (level == NXT_LOG_ALERT + && nxt_memstrn(msg, p, "failed to start a process", + nxt_length("failed to start a process")) != NULL) + { + nxt_router_start_proto_gone_test_alerts++; + } + + nxt_log_error(level, &nxt_main_log, "%s", msg); +} + + +static nxt_int_t +nxt_router_start_proto_gone_test_run(nxt_thread_t *thr, nxt_err_t inject, + nxt_uint_t expected_alerts) +{ + nxt_fd_t pair[2]; + nxt_app_t *app; + nxt_int_t ret; + nxt_log_t *saved_log; + nxt_bool_t app_mutex; + nxt_task_t *task; + nxt_port_t *router_port, *proto_port; + nxt_event_engine_t engine, *saved_engine; + + nxt_log_t log = { + NXT_LOG_INFO, + 0, + nxt_router_start_proto_gone_test_log_handler, + NULL, + NULL + }; + + ret = NXT_ERROR; + router_port = NULL; + proto_port = NULL; + app_mutex = 0; + pair[0] = -1; + pair[1] = -1; + app = &nxt_router_start_proto_gone_test_app; + + task = thr->task; + task->thread = thr; + + if (nxt_slow_path(nxt_port_rpc_init() != NXT_OK)) { + return NXT_ERROR; + } + + /* + * The failed send queues nxt_port_error_handler() on the fast work queue + * of the engine. That handler drops the reference that + * nxt_port_write_msgs() takes for it. + */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + saved_engine = thr->engine; + thr->engine = &engine; + + /* The handler registers its RPC on this port. */ + + router_port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_ROUTER); + if (nxt_slow_path(router_port == NULL)) { + goto done; + } + + /* See src/test/nxt_router_start_fail_test.c. */ + router_port->pair[0] = 0; + router_port->pair[1] = -1; + router_port->socket.fd = -1; + + /* + * The prototype port. It is write-ready and its queue is empty, so + * nxt_port_socket_write2() sends inline and reports the failure to the + * handler. + */ + + proto_port = nxt_port_new(task, 1, nxt_pid, NXT_PROCESS_PROTOTYPE); + if (nxt_slow_path(proto_port == NULL)) { + goto done; + } + + proto_port->pair[0] = -1; + proto_port->pair[1] = -1; + proto_port->socket.fd = -1; + + if (nxt_slow_path(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) != 0)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: socketpair failed"); + goto done; + } + + if (inject == 0) { + /* The prototype has exited: its reader end is closed. */ + nxt_fd_close(pair[0]); + pair[0] = -1; + } + + proto_port->socket.fd = pair[1]; + proto_port->socket.task = task; + proto_port->socket.log = thr->log; + proto_port->socket.write_ready = 1; + proto_port->socket.write = NXT_EVENT_INACTIVE; + proto_port->max_size = 1024; + proto_port->max_share = 1024; + + nxt_memzero(app, sizeof(nxt_app_t)); + + if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { + goto done; + } + + app_mutex = 1; + + nxt_queue_init(&app->ports); + nxt_queue_init(&app->spare_ports); + nxt_queue_init(&app->idle_ports); + nxt_queue_init(&app->ack_waiting_req); + + nxt_str_set(&app->name, "start-proto-gone-test"); + + /* With a prototype port the handler takes the worker start branch. */ + app->proto_port = proto_port; + + /* A real initiator leaves these: the slot and the use of the work. */ + app->pending_processes = 1; + app->use_count = 2; + + /* One process is alive, so the failure path has no requests to fail. */ + app->processes = 1; + + nxt_router_start_proto_gone_test_alerts = 0; + + saved_log = task->log; + task->log = &log; + + if (inject != 0) { + nxt_socketpair_test_send_fail(inject, 1); + } + + nxt_router_start_app_process_handler(task, router_port, app); + + nxt_socketpair_test_send_fail(0, 0); + + task->log = saved_log; + + nxt_router_start_proto_gone_test_drain_wq(&engine.fast_work_queue); + + if (proto_port->socket.error == 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: the START_PROCESS send " + "did not fail (inject %d)", (int) inject); + goto done; + } + + if (app->pending_processes != 0) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: pending_processes %uD " + "leaked (expected 0, inject %d)", + app->pending_processes, (int) inject); + goto done; + } + + if (app->use_count != 1) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: app use_count %d " + "(expected 1, inject %d)", + (int) app->use_count, (int) inject); + goto done; + } + + if (nxt_router_start_proto_gone_test_alerts != expected_alerts) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router start proto gone test: %ui \"failed to start a " + "process\" alerts (expected %ui, send error %d)", + nxt_router_start_proto_gone_test_alerts, + expected_alerts, (int) proto_port->socket.error); + goto done; + } + + ret = NXT_OK; + +done: + + nxt_router_start_proto_gone_test_drain_wq(&engine.fast_work_queue); + + if (proto_port != NULL) { + proto_port->socket.fd = -1; + + nxt_port_use(task, proto_port, -1); + } + + if (pair[0] != -1) { + nxt_fd_close(pair[0]); + } + + if (pair[1] != -1) { + nxt_fd_close(pair[1]); + } + + if (router_port != NULL) { + router_port->pair[0] = -1; + + nxt_port_use(task, router_port, -1); + } + + thr->engine = saved_engine; + + if (app_mutex) { + nxt_thread_mutex_destroy(&app->mutex); + } + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + + return ret; +} + + +static void +nxt_router_start_proto_gone_test_drain_wq(nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *t; + nxt_work_handler_t handler; + + while (wq->head != NULL) { + handler = nxt_work_queue_pop(wq, &t, &obj, &data); + handler(t, obj, data); + } +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 862c4ec67..a7db2dd9a 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -285,6 +285,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_start_proto_gone_test(thr) != NXT_OK) { + return 1; + } + if (nxt_router_proto_wedge_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 7f1b42cce..69a7fa7ad 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -81,6 +81,7 @@ nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); +nxt_int_t nxt_router_start_proto_gone_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_router_proto_death_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_timeout_test(nxt_thread_t *thr); From a743e4b3374d9357c450821329138cb757bfceb6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 13:11:50 +0000 Subject: [PATCH 308/391] fix(prototype): log a QUIT to an exited worker at info, not alert The router stops its own workers before it quits the prototype. A worker can exit before the prototype handles its SIGCHLD. The QUIT to that worker then fails with EPIPE, and the failure was logged as an alert. This is a normal event, so log it at info. The mechanism: - nxt_port_send_msg_t gets a "peer_may_be_gone" flag, in padding. nxt_port_socket_write2() sets it when the caller sends a QUIT. The flag is local to this process. It stays set when the shared queue branch changes the socket message into a READ_QUEUE wake-up. - nxt_port_t gets a "quit_sent" atomic. It is set when a QUIT is sent to the port and is never cleared. A READ_QUEUE wake-up can be pending already when the QUIT goes into the shared queue (notify == 0). That wake-up then covers the QUIT, so its failure is also logged at info. - nxt_socketpair_send_ex() takes the flag. nxt_socketpair_send() calls it with the flag clear. nxt_socketpair_send_error_level() gives the level for a failed send. It also accepts a QUIT header in iob[0]. For a QUIT, EPIPE and ECONNREFUSED (a SOCK_DGRAM pair) are info. All other failures stay an alert. An accepted side effect: after a QUIT, a later failed send on that port to a gone peer is also info. The test nxt_port_fail_test_quit_log_level() has 8 legs. It sends a QUIT or a DATA to a closed peer, at once, from the port queue, or as a wake-up for a shared queue, and checks the level of the "sendmsg() failed" log line. Without the fix it fails with: "a QUIT sent at once to a peer that is gone was logged at level 0, expected 4". Co-Authored-By: Claude Opus 5.5 --- src/nxt_application.c | 6 + src/nxt_port.h | 19 +++ src/nxt_port_socket.c | 31 ++++- src/nxt_socket.h | 2 + src/nxt_socketpair.c | 52 ++++++- src/test/nxt_port_fail_test.c | 251 ++++++++++++++++++++++++++++++++++ 6 files changed, 358 insertions(+), 3 deletions(-) diff --git a/src/nxt_application.c b/src/nxt_application.c index 925bb8be8..6ab9c58c6 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -982,6 +982,12 @@ nxt_proto_quit_children(nxt_task_t *task) port = nxt_process_port_first(process); + /* + * The router stops its own workers before it quits the prototype. + * A worker can exit before the prototype handles its SIGCHLD. + * The QUIT to that worker then fails with EPIPE. + * nxt_socketpair_send_ex() logs this failure at info for a QUIT. + */ nxt_runtime_port_send_quit(task, rt, port); } nxt_queue_loop; diff --git a/src/nxt_port.h b/src/nxt_port.h index a6642a036..df58937dc 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -274,6 +274,15 @@ typedef struct { nxt_port_msg_t port_msg; uint8_t close_fd; /* 1 bit */ uint8_t allocated; /* 1 bit */ + + /* + * The message is a QUIT, so its peer can have exited already. + * A send that fails because the peer is gone is logged at info, not + * as an alert. The flag is local to this process. It is not on the + * wire and not in shared memory. The flag stays set when the socket + * carries only the READ_QUEUE wake-up for a QUIT in the shared queue. + */ + uint8_t peer_may_be_gone; /* 1 bit */ } nxt_port_send_msg_t; #if (NXT_HAVE_UCRED) || (NXT_HAVE_MSGHDR_CMSGCRED) @@ -512,6 +521,16 @@ struct nxt_port_s { nxt_atomic_t rearm_pending; nxt_atomic_t announce; + /* + * A QUIT was sent to this port, so its peer can be gone by now. + * Any thread can set the flag in nxt_port_socket_write2(). The flag + * is never cleared. The sender reads it when a send fails. + * A READ_QUEUE wake-up can be pending already when the QUIT goes into + * the shared queue (notify == 0). That wake-up wakes the peer for + * the QUIT too, so its failure is logged at info like the QUIT's own. + */ + nxt_atomic_t quit_sent; + /* * The pacing of a re-arm that follows a send which failed for want of * kernel memory. diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 73bc66347..542429310 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -317,6 +317,24 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, msg.close_fd = (type & NXT_PORT_MSG_CLOSE_FD) != 0; msg.allocated = 0; + /* + * Set the flag here, from the type the caller asked for. The shared + * queue branch below can change the socket message into a READ_QUEUE + * wake-up, and nxt_port_msg_chk_insert() copies the message. The flag + * is written once, before the message is published under + * port->write_mutex. Only the sender reads it, after that. So the + * flag needs no lock of its own. + */ + msg.peer_may_be_gone = ((type & NXT_PORT_MSG_MASK) == _NXT_PORT_MSG_QUIT); + + /* + * Set port->quit_sent before the QUIT can reach the shared queue. + * A wake-up that is already pending then covers the QUIT as well. + */ + if (msg.peer_may_be_gone) { + (void) nxt_atomic_cmp_set(&port->quit_sent, 0, 1); + } + msg.port_msg.stream = stream; msg.port_msg.pid = nxt_pid; msg.port_msg.reply_port = reply_port; @@ -328,6 +346,15 @@ nxt_port_socket_write2(nxt_task_t *task, nxt_port_t *port, nxt_uint_t type, if (port->queue != NULL && type != _NXT_PORT_MSG_READ_QUEUE) { + /* + * A QUIT goes into the shared queue like any other message. + * A worker that is still in nxt_unit_init() holds only one socket + * message with no queue marker ("too many port socket messages"). + * A QUIT on the socket would be a second one. So the wake-up for + * a QUIT is a plain READ_QUEUE. msg.peer_may_be_gone, set above, + * still marks it as a QUIT. A failed wake-up to a worker that is + * gone is then logged at info. + */ if (fd == -1 && nxt_port_can_enqueue_buf(b)) { qmsg.pm = msg.port_msg; @@ -1185,7 +1212,9 @@ nxt_port_write_msgs(nxt_task_t *task, void *obj, void *data, msg->port_msg.last |= sb.last; msg->port_msg.mf = sb.limit_reached || sb.nmax_reached; - n = nxt_socketpair_send(&port->socket, msg->fd, iov, sb.niov + 1); + n = nxt_socketpair_send_ex(&port->socket, msg->fd, iov, sb.niov + 1, + msg->peer_may_be_gone + || port->quit_sent != 0); if (n > 0) { /* diff --git a/src/nxt_socket.h b/src/nxt_socket.h index 6467e31a1..d2fdebcff 100644 --- a/src/nxt_socket.h +++ b/src/nxt_socket.h @@ -114,6 +114,8 @@ NXT_EXPORT nxt_int_t nxt_socketpair_create(nxt_task_t *task, NXT_EXPORT void nxt_socketpair_close(nxt_task_t *task, nxt_socket_t *pair); NXT_EXPORT ssize_t nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob); +ssize_t nxt_socketpair_send_ex(nxt_fd_event_t *ev, nxt_fd_t *fd, + nxt_iobuf_t *iob, nxt_uint_t niob, nxt_bool_t peer_may_be_gone); NXT_EXPORT ssize_t nxt_socketpair_recv(nxt_fd_event_t *ev, nxt_iobuf_t *iob, nxt_uint_t niob, void *oob); diff --git a/src/nxt_socketpair.c b/src/nxt_socketpair.c index 94ccf9984..da386b92c 100644 --- a/src/nxt_socketpair.c +++ b/src/nxt_socketpair.c @@ -105,9 +105,54 @@ nxt_socketpair_test_send_fail(nxt_err_t err, nxt_uint_t n) #endif +/* + * The level to log a failed send at. A QUIT goes to a process that can + * have exited already, at once or later from the queue. So EPIPE and the + * like on a QUIT are expected. They are logged at the level that + * nxt_socket_error_level() gives, which is info. A SOCK_DGRAM pair (the + * fallback in nxt_socketpair_create()) reports a gone peer as ECONNREFUSED. + * That is info too. All other failures are an alert. + * + * The caller marks a QUIT with "peer_may_be_gone" + * (nxt_port_send_msg_t.peer_may_be_gone). This is the only way to know + * it for the READ_QUEUE wake-up that nxt_port_socket_write2() sends for a + * QUIT in a shared queue. A QUIT header in iob[0] counts as well, because + * nxt_port_write_msgs() sends the port message header first. + */ +static nxt_uint_t +nxt_socketpair_send_error_level(nxt_iobuf_t *iob, nxt_uint_t niob, + nxt_bool_t peer_may_be_gone, nxt_err_t err) +{ + nxt_port_msg_t *msg; + + if (!peer_may_be_gone + && niob != 0 && iob[0].iov_len >= sizeof(nxt_port_msg_t)) + { + msg = iob[0].iov_base; + + peer_may_be_gone = (msg->type == _NXT_PORT_MSG_QUIT); + } + + if (peer_may_be_gone) { + return (err == NXT_ECONNREFUSED) ? NXT_LOG_INFO + : nxt_socket_error_level(err); + } + + return NXT_LOG_ALERT; +} + + ssize_t nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, nxt_uint_t niob) +{ + return nxt_socketpair_send_ex(ev, fd, iob, niob, 0); +} + + +ssize_t +nxt_socketpair_send_ex(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, + nxt_uint_t niob, nxt_bool_t peer_may_be_gone) { ssize_t n; nxt_err_t err; @@ -184,8 +229,11 @@ nxt_socketpair_send(nxt_fd_event_t *ev, nxt_fd_t *fd, nxt_iobuf_t *iob, continue; default: - nxt_alert(ev->task, "sendmsg(%d, %FD, %FD, %ui) failed %E", - ev->fd, fd[0], fd[1], niob, err); + nxt_log(ev->task, + nxt_socketpair_send_error_level(iob, niob, + peer_may_be_gone, err), + "sendmsg(%d, %FD, %FD, %ui) failed %E", + ev->fd, fd[0], fd[1], niob, err); /* * The one exit where the socket really is broken, so it is the diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index ccca0324d..c13c34ca5 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -29,6 +29,12 @@ static nxt_uint_t nxt_port_fail_test_spin(nxt_task_t *task, nxt_msec_t window, nxt_port_t *rearm); #endif static nxt_int_t nxt_port_fail_test_dead_peer(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_quit_log_level(nxt_thread_t *thr); +static nxt_int_t nxt_port_fail_test_send_to_dead_peer(nxt_thread_t *thr, + nxt_uint_t type, nxt_bool_t queued, nxt_bool_t shared, nxt_uint_t then, + nxt_uint_t *level); +static void nxt_cdecl nxt_port_fail_test_log_handler(nxt_uint_t level, + nxt_log_t *log, const char *fmt, ...); static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_error_handler(nxt_thread_t *thr); static nxt_int_t nxt_port_fail_test_mp_baseline(nxt_thread_t *thr); @@ -79,6 +85,10 @@ nxt_port_fail_test(nxt_thread_t *thr) return NXT_ERROR; } + if (nxt_port_fail_test_quit_log_level(thr) != NXT_OK) { + return NXT_ERROR; + } + if (nxt_port_fail_test_wakeup_errno(thr) != NXT_OK) { return NXT_ERROR; } @@ -877,6 +887,247 @@ nxt_port_fail_test_dead_peer(nxt_thread_t *thr) } +/* + * The level a failed send to a gone peer is logged at. A QUIT goes to a + * process that can have exited already. So EPIPE on a QUIT is info, at + * once or later from the port's own queue. On a SOCK_DGRAM pair the error + * is ECONNREFUSED, and that is info too. A failed send of any other + * message is an alert. A message to a port with a shared queue goes + * through that queue. The socket then carries only a READ_QUEUE wake-up. + * The wake-up for a QUIT keeps peer_may_be_gone, so it is info too. This + * also applies when the wake-up waits in port->messages first and fails + * from the write handler. The wake-up for a DATA is an alert. The peer's + * end is closed, so sendmsg() really fails. + */ + +static nxt_uint_t nxt_port_fail_test_sendmsg_level; + + +static void nxt_cdecl +nxt_port_fail_test_log_handler(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...) +{ + if (nxt_strncmp(fmt, "sendmsg(", 8) == 0) { + nxt_port_fail_test_sendmsg_level = level; + } +} + + +static nxt_int_t +nxt_port_fail_test_send_to_dead_peer(nxt_thread_t *thr, nxt_uint_t type, + nxt_bool_t queued, nxt_bool_t shared, nxt_uint_t then, nxt_uint_t *level) +{ + nxt_fd_t pair[2]; + nxt_int_t ret; + nxt_log_t log, *saved_log; + nxt_task_t *task; + nxt_port_t *port; + nxt_event_engine_t engine, *saved_engine; + nxt_port_queue_t *queue; + nxt_port_send_msg_t *msg; + nxt_event_interface_t stub; + + task = thr->task; + task->thread = thr; + + port = nxt_port_fail_test_port(task); + if (nxt_slow_path(port == NULL)) { + return NXT_ERROR; + } + + nxt_memzero(&engine, sizeof(engine)); + nxt_memzero(&stub, sizeof(stub)); + + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + stub.enable_write = nxt_port_fail_test_enable_write; + stub.block_write = nxt_port_fail_test_enable_write; + engine.event = stub; + + saved_engine = thr->engine; + thr->engine = &engine; + + /* Only the "sendmsg() failed" level is of interest. */ + + log = *task->log; + log.level = NXT_LOG_INFO; + log.handler = nxt_port_fail_test_log_handler; + + saved_log = task->log; + task->log = &log; + + nxt_port_fail_test_sendmsg_level = NXT_LOG_DEBUG; + + ret = NXT_ERROR; + queue = NULL; + + /* The production pair: SOCK_SEQPACKET where there is one, else DGRAM. */ + + if (nxt_slow_path(nxt_socketpair_create(task, pair) != NXT_OK)) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: socketpair failed"); + goto done; + } + + if (shared) { + queue = nxt_mp_zalloc(port->mem_pool, sizeof(nxt_port_queue_t)); + if (nxt_slow_path(queue == NULL)) { + goto done; + } + + nxt_port_queue_init(queue); + port->queue = queue; + } + + port->pair[0] = pair[0]; + port->pair[1] = pair[1]; + port->socket.task = task; + port->max_size = 1024; + port->max_share = 1024; + + nxt_port_write_enable(task, port); + + port->socket.log = &log; + port->socket.write = NXT_EVENT_INACTIVE; + port->socket.write_ready = !queued; + + /* The peer is gone before the send. */ + + nxt_fd_close(pair[0]); + port->pair[0] = -1; + + ret = nxt_port_socket_write(task, port, type, -1, 0, 0, NULL); + + if (queue != NULL && queue->nitems != 1) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: the message did not go into the " + "shared queue"); + ret = NXT_ERROR; + goto done; + } + + /* + * Send a second message behind the first one. The shared queue is not + * empty any more. So the second message goes into the queue with no + * wake-up of its own (notify == 0), and the pending wake-up covers it. + */ + if (then != 0) { + ret = nxt_port_socket_write(task, port, then, -1, 0, 0, NULL); + + if (ret != NXT_OK || queue == NULL || queue->nitems != 2) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: the second message did not " + "join the shared queue (%d)", (int) ret); + ret = NXT_ERROR; + goto done; + } + } + + if (queued) { + if (ret != NXT_OK || nxt_queue_is_empty(&port->messages)) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: the message was not queued " + "(%d)", (int) ret); + ret = NXT_ERROR; + goto done; + } + + msg = nxt_queue_link_data(nxt_queue_first(&port->messages), + nxt_port_send_msg_t, link); + + /* What waits is the socket wake-up, not the message itself. */ + + if (shared && msg->port_msg.type != _NXT_PORT_MSG_READ_QUEUE) { + nxt_log_error(NXT_LOG_NOTICE, saved_log, + "port failure test: the queued message is not a " + "READ_QUEUE wake-up (%d)", + (int) msg->port_msg.type); + ret = NXT_ERROR; + goto done; + } + + port->socket.write_ready = 1; + + port->socket.write_handler(task, &port->socket, NULL); + } + + *level = nxt_port_fail_test_sendmsg_level; + + ret = NXT_OK; + +done: + + nxt_port_fail_test_drain_wq(&engine.fast_work_queue); + + /* Not a mapping: nxt_port_close() must not munmap() it. */ + port->queue = NULL; + + nxt_port_close(task, port); + nxt_port_use(task, port, -1); + + task->log = saved_log; + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + + return ret; +} + + +static nxt_int_t +nxt_port_fail_test_quit_log_level(nxt_thread_t *thr) +{ + nxt_uint_t i, level; + + static const struct { + nxt_uint_t type; + nxt_bool_t queued; + nxt_bool_t shared; + nxt_uint_t then; + nxt_uint_t level; + const char *name; + } legs[] = { + { NXT_PORT_MSG_QUIT, 0, 0, 0, NXT_LOG_INFO, "a QUIT sent at once" }, + { NXT_PORT_MSG_QUIT, 1, 0, 0, NXT_LOG_INFO, + "a QUIT sent from the queue" }, + { NXT_PORT_MSG_QUIT, 0, 1, 0, NXT_LOG_INFO, + "a QUIT to a port with a shared queue" }, + { NXT_PORT_MSG_QUIT, 1, 1, 0, NXT_LOG_INFO, + "a deferred QUIT wake-up to a port with a shared queue" }, + { NXT_PORT_MSG_DATA, 0, 0, 0, NXT_LOG_ALERT, "a DATA sent at once" }, + { NXT_PORT_MSG_DATA, 0, 1, 0, NXT_LOG_ALERT, + "a DATA to a port with a shared queue" }, + { NXT_PORT_MSG_DATA, 1, 1, 0, NXT_LOG_ALERT, + "a deferred DATA wake-up to a port with a shared queue" }, + { NXT_PORT_MSG_DATA, 1, 1, NXT_PORT_MSG_QUIT, NXT_LOG_INFO, + "a deferred wake-up that also covers a later QUIT" }, + }; + + for (i = 0; i < nxt_nitems(legs); i++) { + if (nxt_port_fail_test_send_to_dead_peer(thr, legs[i].type, + legs[i].queued, + legs[i].shared, + legs[i].then, &level) + != NXT_OK) + { + return NXT_ERROR; + } + + if (level != legs[i].level) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "port failure test: %s to a peer that is gone was " + "logged at level %ui, expected %ui", legs[i].name, + level, legs[i].level); + return NXT_ERROR; + } + } + + return NXT_OK; +} + + static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr) { From 651cb7718e9bcf1a76c2b6da5598197fb37e547d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 14:10:37 +0000 Subject: [PATCH 309/391] fix(controller): accept the configured control-socket user and group The SO_PEERCRED / getpeereid() check accepted only root and the euid of unitd. When unitd runs as root and the control socket is given to another user or group with --control-user or --control-group, that user passes the file permissions of the socket, but the controller then closes the connection. Resolve the configured owner and group once, with getpwnam_r() and getgrnam_r(), when the control socket is made. The socket is chowned to these ids, and the peer check uses the same ids, so the two cannot differ. nxt_file_chown() now takes ids; the new nxt_file_owner_ids() does the lookup; its buffer grows on ERANGE. nxt_controller_peer_allowed() now also accepts a peer whose uid is the owner, or whose gid or supplementary groups (SO_PEERGROUPS, where the platform has it) include the group. SO_PEERGROUPS needs Linux 4.13 or later; an older kernel returns ENOPROTOOPT, and then only the primary group is checked. The kernel still checks the file mode of the socket on connect(), so this gives no access that the socket permissions do not give. The unitd(8) man page and the comment in nxt_conf_validation.c now describe the new rule. Tests: - src/test/nxt_controller_peer_test.c covers the decision table. - test/test_control_socket_peer.py starts a separate unitd with a delegated socket and connects as "nobody": the configured user and group get a response, an unrelated user is still rejected. EPIPE or ECONNRESET on send counts as a rejected connection, because the controller closes a rejected peer right after accept(). The test unitd runs in its own session, so the fallback cleanup kills the whole process group, and the log of this unitd is printed only when a test fails. This is a forward port of the fix in #144 (1.34 LTS). Co-Authored-By: Claude Opus 5.5 --- CHANGES | 6 + auto/sources | 1 + docs/changes.xml | 9 ++ docs/man/man8/unitd.8.in | 11 +- src/nxt_conf_validation.c | 5 +- src/nxt_controller.c | 137 ++++++++++++++++- src/nxt_file.c | 115 ++++++++------ src/nxt_file.h | 6 +- src/nxt_listen_socket.c | 8 +- src/nxt_runtime.c | 3 + src/nxt_runtime.h | 5 + src/test/nxt_controller_peer_test.c | 80 ++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/test_control_socket_peer.py | 225 ++++++++++++++++++++++++++++ 15 files changed, 553 insertions(+), 63 deletions(-) create mode 100644 src/test/nxt_controller_peer_test.c create mode 100644 test/test_control_socket_peer.py diff --git a/CHANGES b/CHANGES index b8487aaf6..7047d181c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: when unitd ran as root and the control socket was given to + another user or group with --control-user or --control-group, that + user passed the socket file permissions but the controller then + closed the connection. The controller now also accepts the + configured user, and a peer in the configured group. + *) Security: an application could write an unexpected value into an entry of a shared memory queue. The router then retried that entry for ever. Each enqueue and dequeue now stops after one queue size diff --git a/auto/sources b/auto/sources index a9163bf7c..214455678 100644 --- a/auto/sources +++ b/auto/sources @@ -209,6 +209,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_fd_test.c \ src/test/nxt_port_rpc_fd_test.c \ src/test/nxt_port_queued_fd_test.c \ + src/test/nxt_controller_peer_test.c \ src/test/nxt_checked_test.c \ src/test/nxt_conn_close_test.c \ src/test/nxt_router_prepare_msg_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 6ec7ad632..21730159a 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,15 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +when unitd ran as root and the control socket was given to another user or +group with --control-user or --control-group, that user passed the socket file +permissions but the controller then closed the connection. The controller now +also accepts the configured user, and a peer in the configured group. + + + an application could write an unexpected value into an entry of a shared diff --git a/docs/man/man8/unitd.8.in b/docs/man/man8/unitd.8.in index 517d1a957..cb0868bdc 100644 --- a/docs/man/man8/unitd.8.in +++ b/docs/man/man8/unitd.8.in @@ -72,10 +72,15 @@ Sets the group of the UNIX-domain control socket. .Pp On a UNIX-domain control socket, and where the platform provides peer credentials (Linux and the BSDs), the control API additionally requires -the connecting peer's effective user ID to be either 0 (root) or the -user ID +the connecting peer's effective user ID to be 0 (root), the user ID .Nm -runs as; other local users are rejected before the request is read. +runs as, or the user set with +.Fl Fl control-user , +or its group ID to be the group set with +.Fl Fl control-group . +On Linux 4.13 and later, a supplementary group of the peer also matches +.Fl Fl control-group . +Other local users are rejected before the request is read. On those builds the .Fl Fl control-user , .Fl Fl control-group , diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index b1b7b728d..d16953028 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -1407,8 +1407,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[] = { * lets the operator disable every isolation feature here. This is * intentional: any peer who can write to the control socket already * has the same authority as the unitd main process (see the - * SO_PEERCRED check landed in andypost/unit#14 — non-root local - * users are rejected at the socket layer, not by this validator). + * SO_PEERCRED check landed in andypost/unit#14 — local users other + * than root, unitd's user and the --control-user / --control-group + * principals are rejected at the socket layer, not by this validator). * Allow-listing executable paths or forcing isolation = true here * is a deployment policy decision, not a config-schema concern; * deployments needing that should add a wrapping admission gate diff --git a/src/nxt_controller.c b/src/nxt_controller.c index c5688b6be..4ef454d05 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -747,6 +747,17 @@ nxt_runtime_controller_socket(nxt_task_t *task, nxt_runtime_t *rt) const char *path = ls->sockaddr->u.sockaddr_un.sun_path; nxt_fs_mkdir_p_dirname((const u_char *) path, 0755); + + /* + * Resolve --control-user / --control-group once. The same ids + * are used to chown the socket and to check the peer credentials. + */ + if (nxt_file_owner_ids(rt->control_user, rt->control_group, + &rt->control_uid, &rt->control_gid) + != NXT_OK) + { + return NXT_ERROR; + } } #endif @@ -760,6 +771,88 @@ nxt_runtime_controller_socket(nxt_task_t *task, nxt_runtime_t *rt) } +/* + * A peer may use the control API if it is root, runs as unitd's effective + * UID, or is one of the principals the control socket was delegated to with + * --control-user / --control-group. The socket's file permissions are + * still enforced by the kernel on connect(), so this never grants more + * than those permissions already do. + */ + +nxt_bool_t +nxt_controller_peer_allowed(nxt_uid_t uid, nxt_gid_t gid, + const nxt_gid_t *groups, nxt_uint_t ngroups, nxt_uid_t euid, + nxt_uid_t ctl_uid, nxt_gid_t ctl_gid) +{ + nxt_uint_t i; + + if (uid == 0 || uid == euid) { + return 1; + } + + if (ctl_uid != (nxt_uid_t) -1 && uid == ctl_uid) { + return 1; + } + + if (ctl_gid != (nxt_gid_t) -1) { + if (gid == ctl_gid) { + return 1; + } + + for (i = 0; i < ngroups; i++) { + if (groups[i] == ctl_gid) { + return 1; + } + } + } + + return 0; +} + + +#if (NXT_HAVE_UNIX_DOMAIN && NXT_HAVE_UCRED && defined SO_PEERGROUPS) + +static nxt_gid_t * +nxt_controller_peer_groups(nxt_socket_t fd, nxt_uint_t *ngroups) +{ + nxt_err_t err; + nxt_uint_t try; + nxt_gid_t *groups; + socklen_t len; + + len = 64 * sizeof(nxt_gid_t); + + for (try = 0; try < 2; try++) { + groups = nxt_malloc(len); + if (nxt_slow_path(groups == NULL)) { + return NULL; + } + + if (getsockopt(fd, SOL_SOCKET, SO_PEERGROUPS, groups, &len) == 0) { + *ngroups = len / sizeof(nxt_gid_t); + return groups; + } + + err = nxt_errno; + + nxt_free(groups); + + /* + * On ERANGE the kernel stores the required size in len. A kernel + * older than Linux 4.13 returns ENOPROTOOPT: then only the primary + * group is checked. + */ + if (err != ERANGE) { + break; + } + } + + return NULL; +} + +#endif + + static nxt_int_t nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) { @@ -768,8 +861,9 @@ nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) * The control socket is the privilege boundary for the REST API: * filesystem perms on `control.unit.sock` are defense-in-depth, not the * boundary itself. Require the peer's effective UID to match unitd's - * (or be root) so a local user with directory-write permission cannot - * mutate config by hand-crafting connections. + * (or be root, or the configured --control-user / --control-group) so + * a local user with directory-write permission cannot mutate config by + * hand-crafting connections. */ if (c->remote == NULL || c->remote->u.sockaddr.sa_family != AF_UNIX) @@ -779,8 +873,10 @@ nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) #if (NXT_HAVE_UCRED) { - struct ucred cred; - socklen_t len = sizeof(cred); + nxt_bool_t allowed; + struct ucred cred; + socklen_t len = sizeof(cred); + nxt_runtime_t *rt = task->thread->runtime; if (getsockopt(c->socket.fd, SOL_SOCKET, SO_PEERCRED, &cred, &len) != 0 || len < (socklen_t) sizeof(cred)) @@ -789,7 +885,29 @@ nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) return NXT_ERROR; } - if (cred.uid != 0 && cred.uid != nxt_euid) { + allowed = nxt_controller_peer_allowed(cred.uid, cred.gid, NULL, 0, + nxt_euid, rt->control_uid, + rt->control_gid); + +#if (defined SO_PEERGROUPS) + if (!allowed && rt->control_gid != (nxt_gid_t) -1) { + nxt_gid_t *groups; + nxt_uint_t ngroups; + + groups = nxt_controller_peer_groups(c->socket.fd, &ngroups); + + if (groups != NULL) { + allowed = nxt_controller_peer_allowed(cred.uid, cred.gid, + groups, ngroups, + nxt_euid, + rt->control_uid, + rt->control_gid); + nxt_free(groups); + } + } +#endif + + if (!allowed) { nxt_alert(task, "controller: rejecting connection from uid %d " "(unitd uid %d); set socket permissions accordingly", (int) cred.uid, (int) nxt_euid); @@ -801,15 +919,18 @@ nxt_controller_check_peer_cred(nxt_task_t *task, nxt_conn_t *c) #elif (defined(__FreeBSD__) || defined(__APPLE__) || defined(__OpenBSD__) \ || defined(__NetBSD__) || defined(__DragonFly__)) { - uid_t euid; - gid_t egid; + uid_t euid; + gid_t egid; + nxt_runtime_t *rt = task->thread->runtime; if (getpeereid(c->socket.fd, &euid, &egid) != 0) { nxt_alert(task, "controller: getpeereid failed %E", nxt_errno); return NXT_ERROR; } - if (euid != 0 && euid != nxt_euid) { + if (!nxt_controller_peer_allowed(euid, egid, NULL, 0, nxt_euid, + rt->control_uid, rt->control_gid)) + { nxt_alert(task, "controller: rejecting connection from uid %d " "(unitd uid %d)", (int) euid, (int) nxt_euid); return NXT_ERROR; diff --git a/src/nxt_file.c b/src/nxt_file.c index 34a551ce5..bcc930776 100644 --- a/src/nxt_file.c +++ b/src/nxt_file.c @@ -325,83 +325,114 @@ nxt_file_set_access(nxt_file_name_t *name, nxt_file_access_t access) } +/* + * Resolves the user and group names with the reentrant calls. An id stays + * -1 when its name is NULL. The buffer grows on ERANGE, up to 1M. + */ + +#define NXT_FILE_OWNER_BUF_MAX (1024 * 1024) + + nxt_int_t -nxt_file_chown(nxt_file_name_t *name, const char *owner, const char *group) +nxt_file_owner_ids(const char *owner, const char *group, uid_t *uid, + gid_t *gid) { - int err; - char *buf; - long bufsize; - gid_t gid = ~0; - uid_t uid = ~0; + int err; + long size; + char *buf; + struct group grp, *gr; + struct passwd pwd, *pw; - if (owner == NULL && group == NULL) { - return NXT_OK; - } + *uid = (uid_t) -1; + *gid = (gid_t) -1; if (owner != NULL) { - struct passwd pwd, *result; - - bufsize = sysconf(_SC_GETPW_R_SIZE_MAX); - if (bufsize == -1) { - bufsize = 32768; + /* sysconf() returns -1 when the system has no limit. */ + size = sysconf(_SC_GETPW_R_SIZE_MAX); + if (size <= 0) { + size = 16384; } - buf = nxt_malloc(bufsize); - if (buf == NULL) { - return NXT_ERROR; + for ( ;; ) { + buf = nxt_malloc(size); + if (buf == NULL) { + return NXT_ERROR; + } + + err = getpwnam_r(owner, &pwd, buf, size, &pw); + if (err != ERANGE || size >= NXT_FILE_OWNER_BUF_MAX) { + break; + } + + nxt_free(buf); + size *= 2; } - err = getpwnam_r(owner, &pwd, buf, bufsize, &result); - if (result == NULL) { + if (pw == NULL) { nxt_thread_log_alert("getpwnam_r(\"%s\", ...) failed %E %s", - owner, nxt_errno, + owner, err, err == 0 ? "(User not found)" : ""); - goto out_err_free; + nxt_free(buf); + return NXT_ERROR; } - uid = pwd.pw_uid; + *uid = pwd.pw_uid; nxt_free(buf); } if (group != NULL) { - struct group grp, *result; - - bufsize = sysconf(_SC_GETGR_R_SIZE_MAX); - if (bufsize == -1) { - bufsize = 32768; + size = sysconf(_SC_GETGR_R_SIZE_MAX); + if (size <= 0) { + size = 16384; } - buf = nxt_malloc(bufsize); - if (buf == NULL) { - return NXT_ERROR; + for ( ;; ) { + buf = nxt_malloc(size); + if (buf == NULL) { + return NXT_ERROR; + } + + err = getgrnam_r(group, &grp, buf, size, &gr); + if (err != ERANGE || size >= NXT_FILE_OWNER_BUF_MAX) { + break; + } + + nxt_free(buf); + size *= 2; } - err = getgrnam_r(group, &grp, buf, bufsize, &result); - if (result == NULL) { + if (gr == NULL) { nxt_thread_log_alert("getgrnam_r(\"%s\", ...) failed %E %s", - group, nxt_errno, + group, err, err == 0 ? "(Group not found)" : ""); - goto out_err_free; + nxt_free(buf); + return NXT_ERROR; } - gid = grp.gr_gid; + *gid = grp.gr_gid; nxt_free(buf); } + return NXT_OK; +} + + +nxt_int_t +nxt_file_chown(nxt_file_name_t *name, uid_t uid, gid_t gid) +{ + if (uid == (uid_t) -1 && gid == (gid_t) -1) { + return NXT_OK; + } + if (nxt_fast_path(chown((const char *) name, uid, gid) == 0)) { return NXT_OK; } nxt_thread_log_alert("chown(\"%FN\", %l, %l) failed %E", name, - owner != NULL ? (long) uid : -1, - group != NULL ? (long) gid : -1, nxt_errno); - - return NXT_ERROR; - -out_err_free: - nxt_free(buf); + uid != (uid_t) -1 ? (long) uid : -1, + gid != (gid_t) -1 ? (long) gid : -1, nxt_errno); return NXT_ERROR; } diff --git a/src/nxt_file.h b/src/nxt_file.h index aa9a3181d..2b63846b6 100644 --- a/src/nxt_file.h +++ b/src/nxt_file.h @@ -210,8 +210,10 @@ NXT_EXPORT nxt_int_t nxt_file_info(nxt_file_t *file, nxt_file_info_t *fi); NXT_EXPORT nxt_int_t nxt_file_delete(nxt_file_name_t *name); NXT_EXPORT nxt_int_t nxt_file_set_access(nxt_file_name_t *name, nxt_file_access_t access); -NXT_EXPORT nxt_int_t nxt_file_chown(nxt_file_name_t *name, const char *owner, - const char *group); +NXT_EXPORT nxt_int_t nxt_file_owner_ids(const char *owner, const char *group, + uid_t *uid, gid_t *gid); +NXT_EXPORT nxt_int_t nxt_file_chown(nxt_file_name_t *name, uid_t uid, + gid_t gid); NXT_EXPORT nxt_int_t nxt_file_rename(nxt_file_name_t *old_name, nxt_file_name_t *new_name); diff --git a/src/nxt_listen_socket.c b/src/nxt_listen_socket.c index 4fe3e20b6..4a562fda9 100644 --- a/src/nxt_listen_socket.c +++ b/src/nxt_listen_socket.c @@ -127,8 +127,6 @@ nxt_listen_socket_create(nxt_task_t *task, nxt_mp_t *mp, #if (NXT_HAVE_UNIX_DOMAIN) if (family == AF_UNIX) { - const char *user; - const char *group; nxt_runtime_t *rt = thr->runtime; name = (nxt_file_name_t *) sa->u.sockaddr_un.sun_path; @@ -138,10 +136,8 @@ nxt_listen_socket_create(nxt_task_t *task, nxt_mp_t *mp, goto listen_fail; } - user = rt->control_user; - group = rt->control_group; - - if (nxt_file_chown(name, user, group) != NXT_OK) { + /* The ids were resolved once, in nxt_runtime_controller_socket(). */ + if (nxt_file_chown(name, rt->control_uid, rt->control_gid) != NXT_OK) { goto listen_fail; } } diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 57e35b414..e65c27b01 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -68,6 +68,9 @@ nxt_runtime_create(nxt_task_t *task) task->thread->runtime = rt; rt->mem_pool = mp; + rt->control_uid = (nxt_uid_t) -1; + rt->control_gid = (nxt_gid_t) -1; + nxt_thread_mutex_create(&rt->processes_mutex); rt->services = nxt_services_init(mp); diff --git a/src/nxt_runtime.h b/src/nxt_runtime.h index 734a23d7d..15c2ec8e4 100644 --- a/src/nxt_runtime.h +++ b/src/nxt_runtime.h @@ -78,6 +78,8 @@ struct nxt_runtime_s { mode_t control_mode; const char *control_user; const char *control_group; + nxt_uid_t control_uid; /* resolved, or -1 */ + nxt_gid_t control_gid; /* resolved, or -1 */ nxt_str_t certs; nxt_str_t scripts; @@ -141,6 +143,9 @@ NXT_EXPORT nxt_port_t *nxt_runtime_port_find(nxt_runtime_t *rt, nxt_pid_t pid, /* STUB */ nxt_int_t nxt_runtime_controller_socket(nxt_task_t *task, nxt_runtime_t *rt); +nxt_bool_t nxt_controller_peer_allowed(nxt_uid_t uid, nxt_gid_t gid, + const nxt_gid_t *groups, nxt_uint_t ngroups, nxt_uid_t euid, + nxt_uid_t ctl_uid, nxt_gid_t ctl_gid); nxt_str_t *nxt_current_directory(nxt_mp_t *mp); diff --git a/src/test/nxt_controller_peer_test.c b/src/test/nxt_controller_peer_test.c new file mode 100644 index 000000000..b32c3f1a0 --- /dev/null +++ b/src/test/nxt_controller_peer_test.c @@ -0,0 +1,80 @@ +/* + * Copyright (C) FreeUnit + */ + +#include +#include +#include "nxt_tests.h" + + +#define NONE_UID ((nxt_uid_t) -1) +#define NONE_GID ((nxt_gid_t) -1) + + +typedef struct { + nxt_uid_t uid; + nxt_gid_t gid; + const nxt_gid_t *groups; + nxt_uint_t ngroups; + nxt_uid_t euid; + nxt_uid_t ctl_uid; + nxt_gid_t ctl_gid; + nxt_bool_t allowed; +} nxt_controller_peer_test_t; + + +static const nxt_gid_t groups_with_ctl[] = { 10, 20, 500 }; +static const nxt_gid_t groups_without[] = { 10, 20 }; + + +static const nxt_controller_peer_test_t tests[] = { + /* root and unitd's own euid are always allowed */ + { 0, 0, NULL, 0, 1000, NONE_UID, NONE_GID, 1 }, + { 1000, 1000, NULL, 0, 1000, NONE_UID, NONE_GID, 1 }, + + /* unrelated uid without delegation is rejected */ + { 1001, 1001, NULL, 0, 0, NONE_UID, NONE_GID, 0 }, + + /* --control-user alice (uid 1001) with unitd running as root */ + { 1001, 1001, NULL, 0, 0, 1001, NONE_GID, 1 }, + { 1002, 1002, NULL, 0, 0, 1001, NONE_GID, 0 }, + + /* --control-group (gid 500): primary gid match */ + { 1002, 500, NULL, 0, 0, NONE_UID, 500, 1 }, + + /* --control-group: supplementary group match */ + { 1002, 1002, groups_with_ctl, 3, 0, NONE_UID, 500, 1 }, + { 1002, 1002, groups_without, 2, 0, NONE_UID, 500, 0 }, + + /* supplementary groups are ignored without --control-group */ + { 1002, 1002, groups_with_ctl, 3, 0, NONE_UID, NONE_GID, 0 }, +}; + + +nxt_int_t +nxt_controller_peer_test(nxt_thread_t *thr) +{ + nxt_uint_t i; + nxt_bool_t allowed; + const nxt_controller_peer_test_t *t; + + for (i = 0; i < nxt_nitems(tests); i++) { + t = &tests[i]; + + allowed = nxt_controller_peer_allowed(t->uid, t->gid, t->groups, + t->ngroups, t->euid, + t->ctl_uid, t->ctl_gid); + + if (allowed != t->allowed) { + nxt_log_alert(thr->log, "nxt_controller_peer_allowed() test #%ui " + "failed: got %d, expected %d", + i, (int) allowed, (int) t->allowed); + return NXT_ERROR; + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "nxt_controller_peer_allowed() test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index a7db2dd9a..68a50fe88 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -377,6 +377,10 @@ main(int argc, char **argv) } #endif + if (nxt_controller_peer_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CLONE_NEWUSER) if (nxt_clone_creds_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 69a7fa7ad..d20848268 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -107,6 +107,7 @@ nxt_int_t nxt_port_frag_test(nxt_thread_t *thr); nxt_int_t nxt_port_release_test(nxt_thread_t *thr); nxt_int_t nxt_nncq_bound_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); +nxt_int_t nxt_controller_peer_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); nxt_bool_t nxt_test_fd_is_open(nxt_fd_t fd); diff --git a/test/test_control_socket_peer.py b/test/test_control_socket_peer.py new file mode 100644 index 000000000..26001eee9 --- /dev/null +++ b/test/test_control_socket_peer.py @@ -0,0 +1,225 @@ +import grp +import os +import pwd +import shutil +import signal +import socket +import subprocess +import tempfile + +import pytest + +from unit.option import option +from unit.utils import waitforfiles + +prerequisites = {'privileged_user': True} + + +def _peer_ids(): + try: + user = pwd.getpwnam('nobody') + except KeyError: + pytest.skip('requires the "nobody" user') + + for name in ('nogroup', 'nobody'): + try: + return user, grp.getgrnam(name) + except KeyError: + pass + + pytest.skip('requires the "nogroup" or "nobody" group') + + +def _get(sock_path, uid=None, gid=None, groups=None): + """GET / on the control socket from a child with the given credentials. + + Returns the status line, '' when the server closed the connection + without a response, or the connect() error name. + """ + r, w = os.pipe() + pid = os.fork() + + if pid == 0: + os.close(r) + result = b'' + + try: + if groups is not None: + os.setgroups(groups) + if gid is not None: + os.setgid(gid) + if uid is not None: + os.setuid(uid) + + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.settimeout(5) + + try: + s.connect(sock_path) + except OSError as e: + result = f'connect: {e.strerror}'.encode() + else: + # A rejected peer is closed right after accept(), so the + # send can fail with EPIPE or ECONNRESET. That is the same + # result as a close without a response. + try: + s.sendall( + b'GET / HTTP/1.1\r\nHost: localhost\r\n' + b'Connection: close\r\n\r\n' + ) + except OSError: + pass + + data = b'' + + try: + while True: + chunk = s.recv(4096) + if not chunk: + break + data += chunk + except OSError: + pass + + result = data.split(b'\r\n', 1)[0] + + s.close() + + except Exception as e: # pylint: disable=broad-except + result = f'error: {e}'.encode() + + os.write(w, result) + os._exit(0) + + os.close(w) + data = b'' + + while True: + chunk = os.read(r, 4096) + if not chunk: + break + data += chunk + + os.close(r) + os.waitpid(pid, 0) + + return data.decode() + + +@pytest.fixture +def control_unitd(request): + instances = [] + + def run(*args): + tmp = tempfile.mkdtemp(prefix='unit-ctl-test-') + os.chmod(tmp, 0o755) + + sock = f'{tmp}/control.unit.sock' + unitd = f'{option.current_dir}/build/sbin/unitd' + + with open(f'{tmp}/unit.log', 'w', encoding='utf-8') as log: + proc = subprocess.Popen( + [ + unitd, + '--no-daemon', + '--modulesdir', + f'{tmp}/modules', + '--statedir', + f'{tmp}/state', + '--pid', + f'{tmp}/unit.pid', + '--log', + f'{tmp}/unit.log', + '--tmpdir', + tmp, + '--control', + f'unix:{sock}', + *args, + ], + stderr=log, + start_new_session=True, + ) + + instances.append((proc, tmp)) + + assert waitforfiles(sock), 'control socket' + + return sock + + yield run + + for proc, tmp in instances: + proc.send_signal(signal.SIGQUIT) + + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + # unitd has child processes: kill the whole group. + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + proc.wait() + + # Keep the log of this unitd when the test failed. + rep = getattr(request.node, 'rep_call', None) + + if rep is not None and rep.failed: + try: + with open(f'{tmp}/unit.log', encoding='utf-8') as log: + print(f'--- {tmp}/unit.log ---\n{log.read()}') + except OSError: + pass + + shutil.rmtree(tmp, ignore_errors=True) + + +def test_control_socket_peer_default(control_unitd): + user, _ = _peer_ids() + + # World-writable socket without delegation: the kernel allows the + # connect(), but the peer credential check must still reject it. + sock = control_unitd('--control-mode', '0666') + + assert _get(sock).startswith('HTTP/1.1 200'), 'root' + assert ( + _get(sock, uid=user.pw_uid, gid=user.pw_gid, groups=[]) == '' + ), 'unrelated user rejected' + + +def test_control_socket_peer_user(control_unitd): + user, _ = _peer_ids() + + sock = control_unitd( + '--control-user', user.pw_name, '--control-mode', '0600' + ) + + assert _get(sock).startswith('HTTP/1.1 200'), 'root' + assert _get( + sock, uid=user.pw_uid, gid=user.pw_gid, groups=[] + ).startswith('HTTP/1.1 200'), 'control user' + + +def test_control_socket_peer_group(control_unitd): + user, group = _peer_ids() + + sock = control_unitd( + '--control-group', group.gr_name, '--control-mode', '0660' + ) + + other_gid = 12345 if group.gr_gid != 12345 else 12346 + + assert _get( + sock, uid=user.pw_uid, gid=group.gr_gid, groups=[] + ).startswith('HTTP/1.1 200'), 'primary group' + + # Supplementary groups come from SO_PEERGROUPS (Linux 4.13+). + if os.uname().sysname == 'Linux': + assert _get( + sock, uid=user.pw_uid, gid=other_gid, groups=[group.gr_gid] + ).startswith('HTTP/1.1 200'), 'supplementary group' + + assert _get( + sock, uid=user.pw_uid, gid=other_gid, groups=[] + ).startswith('connect:'), 'not in group' From 0da3dbb2541c5b804a2295e2608a8935d565a4a9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 13:25:31 +0000 Subject: [PATCH 310/391] ci: start the slow legs first, cache Rust builds, stop waiting on reads The data is from Build & Test run 36318226905 (PR #448, cf8ce757). The critical path of that run is 12 minutes. The savings are estimates. Only a CI run can measure them. 1. List the slow legs first. The prepare job now gives the legs in this order: python (3.12 first), wasm-wasi-component, java, php, unit, perl, wasm, go, node, ruby. The set of 23 legs does not change. GitHub does not promise to start legs in matrix order. In run 36318226905 python-3.12 was last in the matrix and started last, 194 s after the first leg; it ran 372 s and ended the run. In run 36322425869 of this change it was first in the matrix and started 12th of 23. The order is a hypothesis and has no cost. 2. Stop waiting on keep-alive reads. In test_app_lifecycle_churn_under_keepalive each of 15 reads waited for the full read_timeout of 2 s, because recvall() stops only on EOF or on a timeout. That is 30 s per runtime. recvall() has a new framed option. It stops when the response is complete by its Content-Length or its chunked framing. The test uses it for keep-alive reads only. The assertions and read_timeout do not change. Local run: 30.48 s -> 0.44 s (python), 30.31 s -> 0.27 s (libunit). Estimate: about 60 s less on each leg that runs two runtimes of this file. 3. Balance the python shards by time. Round-robin by count gave pytest 332 s / 243 s / 170 s on python-3.12 / 3.13 / 3.14. The step now gives each shared file of 10 s or more (measured in that run) to the shard with the least load. The other files stay round-robin. test_app_lifecycle.py is one of the measured files, not a fixed part of shard 0. Each shared file gets one shard by construction. The step fails when the time table names a file that is not a shared test file, so a renamed or removed file cannot drop out of the table silently. The union of the files does not change (104 files). Estimate: pytest about 227 s / 241 s / 216 s. 4. Cache the otel Rust build in the core job. "Make unit" took 55 s. The cargo build of src/otel took 44.5 s of it on every run, because ccache covers only C. A new actions/cache step keeps ~/.cargo/registry, ~/.cargo/git and src/otel/target. The key has the OS, the architecture, the rustc version and the hash of src/otel/Cargo.lock. Local run with a warm target: 52 s -> 2 s. Estimate: about 40 s less on the core job. All legs wait for it. 5. Cache the wasm-wasi-component Rust build. "Make wasm-wasi-component" took 189 s: 188 crates, no cache. The same cache pattern keeps src/wasm-wasi-component/target. The key has the rustc version, because "Setup rust" installs the latest stable. The build sets CARGO_INCREMENTAL=0. Estimate: the leg goes from 322 s to about 150 s on a cache hit. The core job also sets CARGO_INCREMENTAL=0 for "Make unit", as the wasm-wasi-component job does. That build uses --release, which already sets it off. The variable keeps it off if the profile changes, so the cached src/otel/target does not grow. New test/test_http_client_framing.py checks HTTP1._response_complete() directly: Content-Length with a full, a partial and an empty body, a bad Content-Length, incomplete headers, chunked bodies with and without the last chunk, a chunk extension, a bad chunk size, and a response with no framing. Without the chunk-extension split, or with no length check, one case fails. test/pytest.ini adds --durations=25. It prints the 25 slowest tests at the end of each run. It does not change the tests. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Fable 5.1 --- .github/workflows/build-test.yml | 186 ++++++++++++++++++++++++++----- test/pytest.ini | 2 +- test/test_app_lifecycle.py | 18 +-- test/test_http_client_framing.py | 45 ++++++++ test/unit/http.py | 67 +++++++++++ 5 files changed, 283 insertions(+), 35 deletions(-) create mode 100644 test/test_http_client_framing.py diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c851f5973..b7a39218d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -58,13 +58,28 @@ jobs: - uses: actions/checkout@v5 - name: Derive build list from pkg/eol.json id: list + # Slow legs come first in the list. All 23 legs are created at + # once. The hosted runner pool is smaller than the matrix, so some + # legs wait for a runner. GitHub does not promise to start legs in + # matrix order. Two runs (36318226905 and 36322425869) showed no + # such order: python-3.12 was last in the old matrix and started + # last; it was first in this matrix and started 12th of 23. The + # order is a hypothesis and has no cost. The set of legs does not + # change. Times from run 36318226905: python-3.12 372 s (pytest + # 332 s; 243 s and 170 s on 3.13 and 3.14), wasm-wasi-component + # 322 s, java 250 to 273 s, php-8.4 255 s. Short legs: perl 109 s, + # wasm 88 s, node about 150 s, ruby about 125 s. run: | builds=$(jq -c ' - ["unit", "perl", "wasm", "wasm-wasi-component"] - + [ .runtimes | to_entries[] - | select(.key | IN("go", "java", "node", "php", "ruby")) - | .key as $rt | .value[] | $rt + "-" + .version ] - + [ .runtimes.python[] | "python-" + .version ] + def legs($rt): [ (.runtimes[$rt] // [])[] | $rt + "-" + .version ]; + legs("python") + + ["wasm-wasi-component"] + + legs("java") + + legs("php") + + ["unit", "perl", "wasm"] + + legs("go") + + legs("node") + + legs("ruby") ' pkg/eol.json) echo "builds=${builds}" >> "$GITHUB_OUTPUT" @@ -153,6 +168,30 @@ jobs: make -j4 -k working-directory: njs + # --otel makes "Make unit" run "cargo rustc" on src/otel. ccache + # covers only C. In run 36318226905 that cargo build took 44.5 s of + # the 55 s of "Make unit", on every run. This step caches the crates + # and the target directory. The target directory is cargo's default, + # src/otel/target: auto/make gives no --target-dir. The key has the + # rustc version: the runner image updates rust, and a different rustc + # rebuilds the target anyway. The otel crate itself is rebuilt on + # every run, because checkout gives its sources a new mtime. Local + # test with a warm target: 52 s -> 2 s. + - name: Read rustc version + id: rustc + run: echo "version=$(rustc -V | tr -c 'A-Za-z0-9.\n' '-')" >> "$GITHUB_OUTPUT" + + - name: Cache the otel Rust build + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + src/otel/target + key: otel-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}-${{ hashFiles('src/otel/Cargo.lock') }} + restore-keys: | + otel-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}- + - name: Configure unit # LD_LIBRARY_PATH is set only for this step so that auto/feature # test binaries load OpenSSL 3.6 at runtime. It is NOT exported @@ -173,7 +212,13 @@ jobs: --cc-opt="-I${OPENSSL36_INCDIR} -I njs/src/ -I njs/build" \ --ld-opt="-L${OPENSSL36_LIBDIR} -Wl,-rpath,${OPENSSL36_LIBDIR} -L njs/build" + # CARGO_INCREMENTAL=0, as for wasm-wasi-component: incremental data + # only makes the cached src/otel/target bigger. This build uses + # --release, which already sets it off. This keeps it off if the + # profile changes. - name: Make unit + env: + CARGO_INCREMENTAL: 0 run: | make -j$(nproc) -k || make @@ -245,13 +290,13 @@ jobs: testpath="test" fi - # A Python leg runs its interpreter-specific suites whole -- that is - # what the three versions are here for -- and takes a round-robin - # share of the rest. Three versions each running the whole - # language-agnostic suite again cost about 12 minutes a leg, twice - # any other language, for tests no interpreter distinguishes. Each - # shared file now runs on one Python version instead of three; the - # union over the legs is unchanged. + # A Python leg runs its interpreter-specific suites whole. That is + # what the three versions are for. It also takes a share of the + # other files, balanced by measured time (see below). Three runs + # of the whole language-agnostic suite cost about 12 minutes per + # leg, twice any other language, for tests that no interpreter + # distinguishes. Each shared file now runs on one Python version + # instead of three. The union over the legs does not change. # # Both lists are built from the files that exist, never from a glob # handed to pytest. There is no test_wsgi_*.py -- WSGI is exercised @@ -284,35 +329,94 @@ jobs: ;; esac - files="" + # Shared files go to the shards by measured time, not by count. + # Round-robin by count gave pytest 332 s / 243 s / 170 s on + # python-3.12 / 3.13 / 3.14 (run 36318226905, PR #448), and the + # 3.12 leg ended the run. The list below has every shared file + # that took 10 s or more in that run, with its time in seconds. + # Each listed file goes to the shard with the least load (the + # first such shard on a tie). All other shared files are + # round-robin, as before. test_app_lifecycle.py took 73 s + # there; 13 s is its time after the keep-alive reads stopped + # waiting for read_timeout. It skips runtimes that are not + # built, so one Python leg is enough for it. The other language + # legs still run it. A file that is not in this list only costs + # balance, not coverage. + heavy=" + test/test_app_start_timeout.py:72 + test/test_settings.py:64 + test/test_otel.py:34 + test/test_proxy.py:27 + test/test_tls.py:22 + test/test_procman.py:21 + test/test_app_lifecycle.py:13 + test/test_idle_close_wait.py:12 + test/test_status.py:11 + test/test_access_log.py:11 + " + + shared="" whole="" - i=0 for f in $(ls test/test_*.py | sort); do case "$f" in test/test_python*|test/test_asgi*|test/test_wsgi*) whole="${whole} ${f}" - continue ;; - test/test_app_lifecycle.py) - continue + *) + shared="${shared} ${f}" ;; esac + done - if [ $((i % versions)) -eq "$shard" ]; then - files="${files} ${f}" + declare -A owner + load=() + for ((s = 0; s < versions; s++)); do + load[s]=0 + done + + # A heavy entry that names no shared test file is a stale table: + # the file was renamed or removed, and its time would silently + # stop counting. Fail instead. + for entry in ${heavy}; do + f="${entry%:*}" + case " ${shared} " in + *" ${f} "*) ;; + *) + echo "::error::python shards: ${f} in the heavy table" \ + "is not a shared test file" >&2 + exit 1 + ;; + esac + + min=0 + for ((s = 1; s < versions; s++)); do + if [ "${load[s]}" -lt "${load[min]}" ]; then + min=$s + fi + done + + owner[$f]=$min + load[min]=$((load[min] + ${entry##*:})) + done + + i=0 + for f in ${shared}; do + if [ -z "${owner[$f]+set}" ]; then + owner[$f]=$((i % versions)) + i=$((i + 1)) fi + done - i=$((i + 1)) + # Each shared file has one owner by construction: the heavy loop + # above, then round-robin. Collect this shard's files. + files="" + for f in ${shared}; do + if [ "${owner[$f]}" -eq "$shard" ]; then + files="${files} ${f}" + fi done testpath="${files}${whole}" - - # app_lifecycle self-skips unbuilt runtimes, so one Python leg is - # enough to carry it; the other language legs still run it as - # before. - if [ "$shard" -eq 0 ]; then - testpath="${testpath} test/test_app_lifecycle.py" - fi fi echo "module=${module}" >> "$GITHUB_OUTPUT" @@ -699,6 +803,29 @@ jobs: tool: cargo-component if: steps.metadata.outputs.module == 'wasm-wasi-component' + # In run 36318226905 "Make wasm-wasi-component" took 189 s: 188 + # crates built from nothing on every run. This step caches the + # crates and the target directory, like the otel build in the core + # job. "Setup rust" above installs the latest stable, so the key has + # the rustc version. This crate is rebuilt on every run, because + # checkout gives its sources a new mtime. + - name: Read rustc version + id: rustc + run: echo "version=$(rustc -V | tr -c 'A-Za-z0-9.\n' '-')" >> "$GITHUB_OUTPUT" + if: steps.metadata.outputs.module == 'wasm-wasi-component' + + - name: Cache the wasm-wasi-component Rust build + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + src/wasm-wasi-component/target + key: wasm-wasi-component-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}-${{ hashFiles('src/wasm-wasi-component/Cargo.lock') }} + restore-keys: | + wasm-wasi-component-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}- + if: steps.metadata.outputs.module == 'wasm-wasi-component' + - name: Configure wasm-wasi-component run: | ./configure wasm-wasi-component @@ -706,7 +833,12 @@ jobs: # bindgen picks clang up from PATH. The libclang-drift check against # new distribution compilers lives in build-distros.yml. + # CARGO_INCREMENTAL=0: incremental data only makes the cached target + # bigger. The release profile already sets it off. This keeps it + # off if the profile changes. - name: Make wasm-wasi-component + env: + CARGO_INCREMENTAL: 0 run: | BINDGEN_EXTRA_CLANG_ARGS="-I../../njs/src -I../../njs/build" \ make wasm-wasi-component diff --git a/test/pytest.ini b/test/pytest.ini index 8952e8c6e..ff8ca7735 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -1,3 +1,3 @@ [pytest] -addopts = -vvv -s --print-log +addopts = -vvv -s --print-log --durations=25 python_functions = test_* diff --git a/test/test_app_lifecycle.py b/test/test_app_lifecycle.py index a48c26277..a75742dc2 100644 --- a/test/test_app_lifecycle.py +++ b/test/test_app_lifecycle.py @@ -198,14 +198,18 @@ def request(self, sock=None, keepalive=False, **kwargs): kwargs['headers'] = dict(KEEPALIVE) kwargs['start'] = True # the server will not close a keep-alive connection, so recvall() - # must be allowed to stop on a short timeout instead of on EOF. + # cannot stop on EOF. framed=True stops the read when the + # response is complete by its Content-Length or its chunked + # framing. Every app in RUNTIMES sends one of the two. Before + # this, every keep-alive read waited for the full read_timeout: + # 15 reads in test_app_lifecycle_churn_under_keepalive, 30 s per + # runtime. # - # Every keep-alive read therefore costs this timeout in full, and - # the same timeout is all that separates "the response is - # complete" from "the app has not answered yet". Two seconds - # buys margin over an ASan-instrumented worker on a loaded runner - # without paying much for it; the startup race that would need - # more than that is closed by waiting for the workers instead. + # read_timeout is now only the limit for an app that has not + # answered. Two seconds gives margin for an ASan worker on a + # loaded runner. A startup race that needs more than that is + # closed by waiting for the workers instead. + kwargs['framed'] = True kwargs['read_timeout'] = 2 if sock is not None: diff --git a/test/test_http_client_framing.py b/test/test_http_client_framing.py new file mode 100644 index 000000000..ecb4f6282 --- /dev/null +++ b/test/test_http_client_framing.py @@ -0,0 +1,45 @@ +import pytest + +from unit.http import HTTP1 + +# HTTP1._response_complete() tells recvall(framed=True) when a keep-alive +# response is complete. These cases pin its framing rules. They need no +# application. + +HEAD = b'HTTP/1.1 200 OK\r\n' + + +@pytest.mark.parametrize( + 'data, complete', + [ + # Content-Length. + (HEAD + b'Content-Length: 5\r\n\r\nhello', True), + (HEAD + b'Content-Length: 5\r\n\r\nhel', False), + (HEAD + b'Content-Length: 0\r\n\r\n', True), + (HEAD + b'content-length: 3 \r\n\r\nabc', True), + (HEAD + b'Content-Length: x\r\n\r\nabc', False), + # The headers are not complete yet. + (HEAD + b'Content-Length: 0\r\n', False), + # Chunked. + ( + HEAD + + b'Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n', + True, + ), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n', False), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\n5\r\nhel', False), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\n0\r\n', False), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\n0\r\n\r\n', True), + # A chunk extension is not part of the size. + ( + HEAD + b'Transfer-Encoding: chunked\r\n\r\n' + b'5;name=value\r\nhello\r\n0\r\n\r\n', + True, + ), + (HEAD + b'Transfer-Encoding: chunked\r\n\r\nzz\r\n', False), + # No framing: the read ends on the timeout. + (HEAD + b'Connection: close\r\n\r\nbody', False), + ], +) +def test_http_client_response_complete(data, complete): + assert HTTP1._response_complete(data) is complete diff --git a/test/unit/http.py b/test/unit/http.py index 1d4004662..446e3ff68 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -128,6 +128,9 @@ def http(self, start_str, **kwargs): if 'read_buffer_size' in kwargs: recvall_kwargs['buff_size'] = kwargs['read_buffer_size'] + if kwargs.get('framed'): + recvall_kwargs['framed'] = True + resp = self.recvall(sock, **recvall_kwargs).decode( encoding, errors='ignore' ) @@ -212,6 +215,7 @@ def recvall(self, sock, **kwargs): timeout = kwargs.get('read_timeout', timeout_default) buff_size = kwargs.get('buff_size', 4096) + framed = kwargs.get('framed', False) data = b'' while True: @@ -237,8 +241,71 @@ def recvall(self, sock, **kwargs): if not part: break + if framed and self._response_complete(data): + break + return data + @staticmethod + def _response_complete(data): + """Return True when data holds one complete response by its framing. + + recvall(framed=True) uses it on a keep-alive socket. The server + does not close that socket, so without this the read ends only on + read_timeout. The function knows Content-Length and chunked. For + a response with neither, or one that does not parse, it returns + False and the read ends on the timeout, as before. Do not use it + for HEAD, 1xx, 204 or 304 responses: they have no body, whatever + their headers say. + """ + end = data.find(b'\r\n\r\n') + if end < 0: + return False + + body = data[end + 4 :] + length = None + chunked = False + + for line in data[:end].split(b'\r\n')[1:]: + name, _, value = line.partition(b':') + name = name.strip().lower() + value = value.strip() + + if name == b'content-length': + try: + length = int(value) + except ValueError: + return False + + elif name == b'transfer-encoding': + chunked = value.lower().endswith(b'chunked') + + if chunked: + while True: + eol = body.find(b'\r\n') + if eol < 0: + return False + + try: + size = int(body[:eol].split(b';')[0], 16) + except ValueError: + return False + + if size == 0: + # Unit sends no trailers: the last chunk is "0" CRLF CRLF. + return body[eol + 2 : eol + 4] == b'\r\n' + + need = eol + 2 + size + 2 + if len(body) < need: + return False + + body = body[need:] + + if length is not None: + return len(body) >= length + + return False + def _resp_to_dict(self, resp): m = re.search(r'(.*?\x0d\x0a?)\x0d\x0a?(.*)', resp, re.M | re.S) From 296ebda93efbe791d39873eb7b9af8491d854faf Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 17:20:15 +0200 Subject: [PATCH 311/391] http: select a compressor that can be applied, and let "*" mean the unnamed codings nxt_http_comp_select_compressor() took the coding with the highest weight and stopped. Two things were wrong with that. "min_length" is an option of each compressor. When the coding with the highest weight is below its own minimum, the next coding can still be above its minimum. With gzip at "min_length" 1000 and deflate at 0, a 2-byte response can be served as deflate. The selection chose gzip on its weight. nxt_http_comp_apply_compression() then found gzip below its minimum, and the response went out uncompressed. The selection now skips a coding below its own "min_length". Both functions call nxt_http_comp_below_min_len(), so the selected coding is the applied coding. A client that refused identity and named only codings below their minimum now falls back to identity. The client refused identity, so the selection answers 406 instead of the bytes that the client refused. The next commit gives the same answer when the "types" rule withdraws the selected coding. The wildcard was read as identity. RFC 9110 Sect. 12.5.3 says that "*" "matches any available content coding not explicitly listed in the field". So "*" stands for each enabled coding that the client did not name, and for identity, at its own weight. Read as identity only, the wildcard could never select a compressor: GET /big.css (7500 bytes, text/css) Accept-Encoding: identity;q=0, *;q=1 -> 406, with gzip enabled, applicable and asked for The wildcard is now recorded while the field is read, and applied after the loop. The wildcard never matches a named coding. A named coding keeps its own weight at any place in the field: "identity;q=0, *;q=0.5, gzip;q=0.1" selects gzip at 0.1 when gzip is the only compressor, and deflate at 0.5 when deflate is enabled and takes the weight of the wildcard. The comparison against the current best is strict, so a named coding wins a tie: "gzip, *" now sends gzip. Before, it sent identity. Among the codings that the wildcard stands for, identity is taken first, so a bare "*" still sends the own bytes of the response. A coding below its "min_length" is skipped here too: the wildcard must not offer a coding that cannot be applied. A repeated "compressors" entry cannot be reached by name, and the wildcard does not reach it either. The wildcard test on the raw token moves out of nxt_http_comp_compressor_lookup_enabled() to the caller, before the lookup. The lookup now searches only the enabled array. For a client that accepts identity, the only visible change is the tie: "gzip, *" is gzip. Each other change here changes only the answer to a client that refused identity. The changelog entry of the previous fix (a byte-range request from a client that refused identity) had no element of its own in docs/changes.xml. It was part of the ASGI entry before it. CHANGES had no blank line before that entry. Both are corrected here. Measured with gcc 13.3 and PHP 8.4.19: the build has 0 warnings, ./build/tests passes, and test_static.py, test_static_compression.py and test_php_compression.py give 94 passed, 5 skipped at this commit. Without the "min_length" skip in the selection loop, test_static_compression_min_length_is_per_compressor and the two identity_refused_below_min_length tests fail. With the wildcard read as identity again, test_static_compression_wildcard_stands_for_every_coding, test_static_compression_wildcard_skips_inapplicable_coding and test_php_compression_wildcard_offers_a_coding fail. With no Accept-Encoding field, nxt_http_comp_select_compressor() now returns identity at once. Before, it gave the NULL start of the absent field to nxt_str_cstrz(), and memcpy() got a NULL source. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 20 +++ docs/changes.xml | 29 +++++ src/nxt_http_compression.c | 217 ++++++++++++++++++++++++++------ test/test_php_compression.py | 62 +++++++++ test/test_static_compression.py | 166 +++++++++++++++++++++++- 5 files changed, 454 insertions(+), 40 deletions(-) diff --git a/CHANGES b/CHANGES index 7047d181c..e0d84d006 100644 --- a/CHANGES +++ b/CHANGES @@ -545,6 +545,7 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 than that is still accepted when nothing else is waiting. A fragmented message also left a count behind that lowered the limit applied to the next message. + *) Bugfix: a byte-range request from a client that refused the identity coding is no longer answered with identity bytes. A range is served as identity, so "Range" together with "Accept-Encoding: @@ -559,6 +560,25 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 single comma-joined value RFC 9110 defines them to be; previously only the first was consulted. + *) Bugfix: "*" in "Accept-Encoding" now stands for each enabled + compressor that the client did not name. Before, it stood for the + identity coding only. "identity;q=0, *;q=1" was answered "406 Not + Acceptable" with gzip enabled and applicable, because the wildcard + could select only identity, and the same field refused identity. A + coding that the field names keeps its own weight. The wildcard does + not match it, and the named coding wins a tie: "gzip, *" now sends + gzip. Before, it sent the response uncompressed. + + *) Bugfix: a compressor below its own "min_length" is no longer selected + before a compressor with a lower weight that can be applied. + "min_length" belongs to each compressor. With gzip at 1000 and + deflate at 0, a 100-byte response was sent uncompressed, because + gzip had the higher weight. A client that refused the identity + coding got the same uncompressed bytes. Such a client now gets + deflate. When each compressor that the client accepts is below its + "min_length", the client gets "406 Not Acceptable" instead of the + bytes it refused. + *) Bugfix: a malformed weight in "Accept-Encoding" is no longer read as a refusal. The qvalue was converted with strtod(), which takes no digits at all from "identity;q=" and from "identity;q=abc" and reports zero, diff --git a/docs/changes.xml b/docs/changes.xml index 21730159a..9c7091216 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -721,6 +721,11 @@ connection to 10 MB of payload that the application has not received yet receive()); a single frame larger than that is still accepted when nothing else is waiting. A fragmented message also left a count behind that lowered the limit applied to the next message. + + + + + a byte-range request from a client that refused the identity coding is no longer answered with identity bytes. A range is served as identity, so Range together with "Accept-Encoding: identity;q=0" returned a 206 carrying @@ -733,6 +738,30 @@ first was consulted. + + +"*" in Accept-Encoding now stands for each enabled compressor that the client +did not name. Before, it stood for the identity coding only. +"identity;q=0, *;q=1" was answered 406 with gzip enabled and applicable, +because the wildcard could select only identity, and the same field refused +identity. A coding that the field names keeps its own weight. The wildcard +does not match it, and the named coding wins a tie: "gzip, *" now sends gzip. +Before, it sent the response uncompressed. + + + + + +a compressor below its own min_length is no longer selected before a +compressor with a lower weight that can be applied. min_length belongs to +each compressor. With gzip at 1000 and deflate at 0, a 100-byte response was +sent uncompressed, because gzip had the higher weight. A client that refused +the identity coding got the same uncompressed bytes. Such a client now gets +deflate. When each compressor that the client accepts is below its +min_length, the client gets 406 instead of the bytes it refused. + + + a malformed weight in Accept-Encoding is no longer read as a refusal. The diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index a69771111..880128dcb 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -535,16 +535,6 @@ static nxt_uint_t nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, const nxt_str_t *token) { - /* - * The wildcard is the whole token, not merely its first character: - * "*" is a tchar, so "*foo" is a legal (and unknown) coding name, and - * matching on the first byte alone made it stand for every coding. - */ - - if (token->length == 1 && token->start[0] == '*') { - return NXT_HTTP_COMP_SCHEME_IDENTITY; - } - /* * RFC 9110 Sect. 8.4.1: a content coding is a token, and tokens are * compared case-insensitively. "Identity;q=0" and "GZIP" are as valid @@ -676,28 +666,78 @@ nxt_http_comp_accept_encoding(nxt_http_request_t *r, nxt_str_t *value) } +/* + * The length that the compressor decision uses. Returns -1 when the length + * of the body is not known yet. + */ + +static nxt_off_t +nxt_http_comp_resp_length(const nxt_http_request_t *r) +{ + if (r->resp.content_length_n > -1) { + return r->resp.content_length_n; + } + + if (r->resp.content_length != NULL) { + return strtol((char *) r->resp.content_length->value, NULL, 10); + } + + return -1; +} + + +/* + * Tells whether "min_length" refuses this response. A body of unknown + * length is compressed, so it is never below the minimum. + * + * nxt_http_comp_select_compressor() and nxt_http_comp_apply_compression() + * both call this, and they must get the same answer. The first decides + * which coding is selected. The second decides whether the selected coding + * is applied. + */ + +static bool +nxt_http_comp_below_min_len(nxt_off_t clen, + const nxt_http_comp_compressor_t *compressor) +{ + return clen > -1 && clen < compressor->opts.min_len; +} + + static nxt_int_t nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, nxt_http_request_t *r, const nxt_str_t *token, bool *identity_refused) { /* - * "identity_allowed" carries what the wildcard said; "identity_named" - * and "identity_named_ok" carry what an explicit "identity" token said. - * They are kept apart because the explicit one wins: in - * "gzip, identity;q=0.5, *;q=0" the client refused everything it did not - * name and then named identity as acceptable, so identity is acceptable. - * Collapsing the two lets the wildcard veto a coding the client allowed. + * What the field said about identity. "identity_named" and + * "identity_named_ok" record an explicit "identity" token. The + * wildcard pair below records "*". They are kept apart because the + * explicit token wins. "named" records the enabled codings that the + * field listed. The wildcard does not stand for these codings. */ bool identity_allowed = true; bool identity_named = false; bool identity_named_ok = false; + bool wildcard_seen = false; char *str, *tkn, *tail, *cur; double weight = 0.0; + double wildcard_qval = 0.0; + uint32_t named = 0; + nxt_off_t clen = nxt_http_comp_resp_length(r); nxt_int_t idx = NXT_HTTP_COMP_SCHEME_IDENTITY; *identity_refused = false; + /* + * No Accept-Encoding field: identity is acceptable. This also keeps + * nxt_str_cstrz() away from the NULL start of an absent field. + */ + + if (token->length == 0) { + return NXT_HTTP_COMP_SCHEME_IDENTITY; + } + str = nxt_str_cstrz(r->mem_pool, token); if (str == NULL) { return NXT_HTTP_COMP_SCHEME_IDENTITY; @@ -724,11 +764,11 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, *tail = '\0'; while ((tkn = strsep(&str, ","))) { - char *qptr; - double qval = 1.0; - nxt_str_t enc; - nxt_uint_t ecidx; - nxt_http_comp_scheme_t scheme; + bool wildcard; + char *qptr; + double qval = 1.0; + nxt_str_t enc; + nxt_uint_t ecidx; qptr = nxt_http_comp_find_weight(tkn); if (qptr != NULL && !nxt_http_comp_parse_weight(qptr + 3, &qval)) { @@ -738,20 +778,62 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, enc.start = (u_char *)tkn; enc.length = qptr != NULL ? (size_t)(qptr - tkn) : strlen(tkn); + /* + * The wildcard is the whole token, not merely its first character: + * "*" is a tchar, so "*foo" is a legal (and unknown) coding name, and + * matching on the first byte alone made it stand for every coding. + */ + + wildcard = (enc.length == 1 && enc.start[0] == '*'); + + /* + * The wildcard names no coding of its own. It is only recorded + * here. The pass after the loop turns it into candidates. + */ + + if (wildcard) { + wildcard_seen = true; + wildcard_qval = qval; + continue; + } + ecidx = nxt_http_comp_compressor_lookup_enabled(conf, &enc); if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) { continue; } - scheme = conf->enabled[ecidx].type->scheme; + /* A built configuration holds identity at index 0. */ + + if (ecidx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + identity_named = true; + identity_named_ok = (qval != 0.0); - if (scheme == NXT_HTTP_COMP_SCHEME_IDENTITY) { - if (enc.length == 1 && enc.start[0] == '*') { - identity_allowed = (qval != 0.0); + } else { + /* + * Record the coding as named, whether or not it can be applied: + * the wildcard stands only for the codings that the field did + * not name. An index past the width of the mask is left out of + * the wildcard. That is the safe direction. "compressors" can + * repeat an encoding, so the index is not bounded by the number + * of distinct codings. + */ + + if (ecidx < sizeof(named) * 8) { + named |= (uint32_t) 1 << ecidx; + } - } else { - identity_named = true; - identity_named_ok = (qval != 0.0); + /* + * A coding below its own "min_length" is never applied. So it + * cannot replace a refused identity, and it must not outrank a + * coding that can be applied. "min_length" is set per + * compressor. With gzip at 1000 and zstd at 0, a 100-byte + * response can be served as zstd. Before, gzip was selected on + * its weight alone, and a request that could be satisfied was + * refused. + */ + + if (nxt_http_comp_below_min_len(clen, &conf->enabled[ecidx])) { + continue; } } @@ -763,12 +845,79 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, weight = qval; } + /* + * Sect. 12.5.3: identity "is acceptable by default unless specifically + * excluded by the Accept-Encoding header field stating either + * 'identity;q=0' or '*;q=0' without a more specific entry for + * 'identity'". So the explicit token wins. In + * "gzip, identity;q=0.5, *;q=0" the client refused everything it did + * not name, and then named identity as acceptable. So identity is + * acceptable. + */ + if (identity_named) { identity_allowed = identity_named_ok; + + } else if (wildcard_seen) { + identity_allowed = (wildcard_qval != 0.0); } *identity_refused = !identity_allowed; + /* + * Sect. 12.5.3: "*" matches any available content coding not explicitly + * listed in the field. So it stands for each enabled coding that the + * client did not name, and for identity, at its own weight. Before, + * the wildcard was read as identity only. Then it could not select a + * compressor, and "identity;q=0, *;q=1" was answered 406 although gzip + * was enabled and applicable. + * + * This pass runs after the loop. So a named coding keeps its own + * weight at any place in the field. The comparison is strict, so a + * named coding wins a tie with the wildcard. Identity is taken before + * any compressor: all of these have the one wildcard weight, and + * identity needs nothing applied. + */ + + if (wildcard_qval > weight) { + if (!identity_named) { + idx = NXT_HTTP_COMP_SCHEME_IDENTITY; + weight = wildcard_qval; + + } else { + for (nxt_uint_t i = 1; i < conf->nr_enabled; i++) { + const nxt_str_t *tok = &conf->enabled[i].type->token; + + if (i >= sizeof(named) * 8 + || (named & ((uint32_t) 1 << i)) != 0) + { + continue; + } + + /* + * A lookup by name finds only the first entry for a coding. + * So a repeated "compressors" entry cannot be reached by + * name. The wildcard must not reach it either. Otherwise + * "*" would run a coding with options that no named request + * can ask for. + */ + + if (nxt_http_comp_compressor_lookup_enabled(conf, tok) != i) { + continue; + } + + if (nxt_http_comp_below_min_len(clen, &conf->enabled[i])) { + continue; + } + + idx = i; + weight = wildcard_qval; + + break; + } + } + } + if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY && !identity_allowed) { return -1; } @@ -1208,7 +1357,6 @@ nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) { int err; nxt_int_t idx; - nxt_off_t min_len; nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); nxt_http_comp_conf_t *conf; nxt_http_comp_compressor_t *compressor; @@ -1226,16 +1374,9 @@ nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) conf = nxt_http_comp_request_conf(r); compressor = &conf->enabled[idx]; - if (r->resp.content_length_n > -1) { - ctx->resp_clen = r->resp.content_length_n; - } else if (r->resp.content_length != NULL) { - ctx->resp_clen = - strtol((char *)r->resp.content_length->value, NULL, 10); - } - - min_len = compressor->opts.min_len; + ctx->resp_clen = nxt_http_comp_resp_length(r); - if (ctx->resp_clen > -1 && ctx->resp_clen < min_len) { + if (nxt_http_comp_below_min_len(ctx->resp_clen, compressor)) { return NXT_OK; } diff --git a/test/test_php_compression.py b/test/test_php_compression.py index 171019496..edfb49fd8 100644 --- a/test/test_php_compression.py +++ b/test/test_php_compression.py @@ -295,3 +295,65 @@ def test_php_compression_identity_refused(): assert status == 406, 'no representation left for the application body' assert 'Content-Encoding' not in headers + + +def test_php_compression_wildcard_offers_a_coding(): + # The wildcard reaches the application path too. Sect. 12.5.3: "*" + # matches any available content coding not explicitly listed. So this + # client refuses the own bytes of the response and accepts everything + # else, gzip included. Before, "*" was read as identity only. Then + # nothing was acceptable, and the router answered 406 to a request that + # it could serve. + client.load('comp_large_body') + configure_compression() + + status, headers, body = raw_get('/', 'identity;q=0, *;q=1') + + assert status == 200, 'the wildcard offers gzip' + assert headers.get('Content-Encoding') == 'gzip', 'encoding advertised' + assert gzip.decompress(body) == b'A' * 100000, 'round-trips' + + # A coding that the field names keeps its own weight, and the wildcard + # does not match it. So gzip is worth 0.1 here, and gzip is still sent. + status, headers, body = raw_get('/', 'identity;q=0, *;q=0.5, gzip;q=0.1') + + assert status == 200, 'an explicitly named gzip is still acceptable' + assert headers.get('Content-Encoding') == 'gzip', 'at its own weight' + assert gzip.decompress(body) == b'A' * 100000 + + +def test_php_compression_identity_refused_below_min_length(): + # gzip is acceptable but never applied: "min_length" is above this body. + # So the response would fall back to the identity that the client + # refused. The answer is 406, and still not 503. + client.load('comp_small_body') + + # client.load() installs a configuration without a "settings" object. + # So a PUT to settings/http/compression is rejected. The whole + # "settings" object is put, and the result is checked. Before, the PUT + # failed silently, and this test passed through the no-compressor path. + assert 'success' in client.conf( + { + "http": { + "compression": { + "types": ["text/html*"], + "compressors": [ + {"encoding": "gzip", "min_length": 1000} + ], + } + } + }, + 'settings', + ), 'gzip with min_length 1000' + + status, headers, _ = raw_get('/', 'gzip, identity;q=0') + + assert status == 406, 'a below-minimum gzip is not available' + assert 'Content-Encoding' not in headers + + # The same response to a client that takes identity is unchanged. + status, headers, body = raw_get('/', 'gzip') + + assert status == 200, 'identity is acceptable here' + assert 'Content-Encoding' not in headers + assert body == b'A' * 64, 'the body, uncompressed' diff --git a/test/test_static_compression.py b/test/test_static_compression.py index bfbeda597..2935c8080 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -1,4 +1,5 @@ import gzip +import zlib from pathlib import Path import pytest @@ -15,6 +16,10 @@ def setup_method_fixture(temp_dir): Path(f'{assets_dir}/big.css').write_text( 'body{color:red}' * 500, encoding='utf-8' ) + Path(f'{assets_dir}/tiny.css').write_text('ok', encoding='utf-8') + # Above the 10-byte "min_length" below. So a 406 on this file can come + # only from the media type, never from the length. + Path(f'{assets_dir}/raw').write_text('raw' * 10, encoding='utf-8') assert 'success' in client.conf( { @@ -337,10 +342,10 @@ def test_static_compression_vary_merge_identity(temp_dir, configured, expected): # needs a language-module test, not a static one. -def _raw_get(**headers): +def _raw_get(url='/big.css', **headers): # Raw bytes: the body has to be decompressed, so it must not be decoded. raw = client.get( - url='/big.css', + url=url, headers={'Host': 'localhost', 'Connection': 'close', **headers}, encoding='latin-1', read_buffer_size=1024 * 1024, @@ -379,6 +384,163 @@ def test_static_compression_range_identity_refused(temp_dir): assert gzip.decompress(body) == data, 'the whole file, correctly coded' +def test_static_compression_identity_refused_below_min_length(): + # "min_length" is 10 and tiny.css is two bytes. So the gzip that made + # this request serveable is never applied, and the fallback is the + # identity that the client refused. The answer is 406, with a Range and + # without one: the Range is not what makes the request unserveable. + for extra in ({}, {'Range': 'bytes=0-1'}): + status, headers, _ = _raw_get( + '/tiny.css', + **{'Accept-Encoding': 'gzip, identity;q=0', **extra}, + ) + assert status == 406, 'a below-minimum gzip is not available' + assert 'Content-Encoding' not in headers + assert 'Content-Range' not in headers + + +def test_static_compression_min_length_is_per_compressor(): + # "min_length" belongs to each compressor. When the coding with the + # highest weight is below its own minimum, the next coding can still be + # above its minimum. gzip is at 1000 and deflate at 0, and tiny.css is + # two bytes: it is serveable as deflate. Before, gzip was selected on + # its weight alone, and the response was identity. For a client that + # refused identity, that was a 406 for a request that can be satisfied. + assert 'success' in client.conf( + { + "types": ["text/css"], + "compressors": [ + {"encoding": "gzip", "min_length": 1000}, + {"encoding": "deflate", "min_length": 0}, + ], + }, + 'settings/http/compression', + ), 'two compressors, different minimums' + + for extra in ({}, {'Accept-Encoding': 'gzip, deflate;q=0.5, identity;q=0'}): + headers = {'Accept-Encoding': 'gzip, deflate;q=0.5', **extra} + + status, hdrs, body = _raw_get('/tiny.css', **headers) + + assert status == 200, f'deflate can serve it: {headers}' + assert hdrs.get('Content-Encoding') == 'deflate', 'the eligible coding' + assert zlib.decompress(body) == b'ok', 'round-trips' + + # The weight order still decides between codings that can all be applied. + status, hdrs, _ = _raw_get( + '/big.css', **{'Accept-Encoding': 'gzip, deflate;q=0.5'} + ) + + assert status == 200, 'both are above their minimum here' + assert hdrs.get('Content-Encoding') == 'gzip', 'the highest weight wins' + + +def test_static_compression_wildcard_stands_for_every_coding(temp_dir): + # Sect. 12.5.3: "The asterisk '*' symbol in an Accept-Encoding field + # matches any available content coding not explicitly listed in the + # field." So "*" is not a synonym for identity. It offers each enabled + # coding that the client did not name, at its own weight. + # + # Before, "*" was read as identity only. Then it could not select a + # compressor. This request refuses identity and accepts everything else, + # and it was answered 406 although gzip was enabled, above its + # "min_length" and inside "types". + data = Path(f'{temp_dir}/assets/big.css').read_bytes() + + status, headers, body = _raw_get( + **{'Accept-Encoding': 'identity;q=0, *;q=1'} + ) + + assert status == 200, 'the wildcard offers gzip' + assert headers.get('Content-Encoding') == 'gzip', 'served as gzip' + assert gzip.decompress(body) == data, 'the whole file, correctly coded' + + # The wildcard does not match a coding that the field names. That coding + # keeps its own weight, however low. gzip is listed at 0.1, so gzip is + # worth 0.1. gzip is the only compressor, so it is still the only coding + # left when identity is refused. + status, headers, body = _raw_get( + **{'Accept-Encoding': 'identity;q=0, *;q=0.5, gzip;q=0.1'} + ) + + assert status == 200, 'an explicitly named gzip is still acceptable' + assert headers.get('Content-Encoding') == 'gzip', 'at its own weight' + assert gzip.decompress(body) == data + + # The wildcard does not bring back a coding that the field refused. + # gzip is out, and "*" offers identity here. + status, headers, body = _raw_get(**{'Accept-Encoding': 'gzip;q=0, *;q=1'}) + + assert status == 200, 'the wildcard still offers identity' + assert 'Content-Encoding' not in headers, 'gzip was refused by name' + assert body == data + + # A named coding wins a tie with the wildcard. The wildcard is the + # weaker statement, about a coding that the client did not name. + for spelling in ('gzip, *', 'gzip;q=0.5, *;q=0.5'): + status, headers, _ = _raw_get(**{'Accept-Encoding': spelling}) + + assert status == 200, f'tie with the wildcard: {spelling!r}' + assert headers.get('Content-Encoding') == 'gzip', 'the named coding' + + # When the field names no coding, the wildcard covers identity too, and + # identity takes the tie: nothing has to be applied to send it. + for spelling in ('*', '*;q=1'): + status, headers, body = _raw_get(**{'Accept-Encoding': spelling}) + + assert status == 200, f'the bare wildcard: {spelling!r}' + assert 'Content-Encoding' not in headers, 'identity takes the tie' + assert body == data + + +def test_static_compression_wildcard_skips_inapplicable_coding(temp_dir): + # The wildcard can stand only for a coding that is really applied. gzip + # is the only compressor, and tiny.css is below its "min_length". So "*" + # can offer only the identity that this client refused: 406, not a 200 of + # the bytes it refused. + status, headers, _ = _raw_get( + '/tiny.css', **{'Accept-Encoding': 'identity;q=0, *;q=1'} + ) + + assert status == 406, 'a below-minimum gzip is not available to "*"' + assert 'Content-Encoding' not in headers + + # "min_length" is set per compressor, so the wildcard must look past the + # first one. gzip is at 1000 and deflate at 0, and tiny.css is two + # bytes: "*" offers deflate, and the request is serveable. + assert 'success' in client.conf( + { + "types": ["text/css"], + "compressors": [ + {"encoding": "gzip", "min_length": 1000}, + {"encoding": "deflate", "min_length": 0}, + ], + }, + 'settings/http/compression', + ), 'two compressors, different minimums' + + status, headers, body = _raw_get( + '/tiny.css', **{'Accept-Encoding': 'identity;q=0, *;q=1'} + ) + + assert status == 200, 'the wildcard reaches the eligible coding' + assert headers.get('Content-Encoding') == 'deflate', 'the one that applies' + assert zlib.decompress(body) == b'ok', 'round-trips' + + # A coding that the wildcard offers at a higher weight wins against a + # coding that the field names at a lower weight: deflate at the wildcard + # weight 0.5 beats the named gzip at 0.1. + status, headers, body = _raw_get( + **{'Accept-Encoding': 'identity;q=0, *;q=0.5, gzip;q=0.1'} + ) + + assert status == 200 + assert headers.get('Content-Encoding') == 'deflate', 'the higher weight' + assert zlib.decompress(body) == Path( + f'{temp_dir}/assets/big.css' + ).read_bytes() + + def test_static_compression_range_identity_refused_guards(temp_dir): # The cases either side of it, which must not move. size = Path(f'{temp_dir}/assets/big.css').stat().st_size From 20b93ac31f0fd5073c37e119b70c4266e1fcd79b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 17:20:15 +0200 Subject: [PATCH 312/391] http: answer 406 where no coding can replace a refused identity A client that sends "identity;q=0" does not take the own bytes of the response. With a compressor configured, the request was accepted because of a coding that then never ran, and the client got identity. Reproduced with gzip configured for text/css: GET /raw (no media type) Accept-Encoding: gzip, identity;q=0 -> 200, no Content-Encoding, body "raw" nxt_http_comp_check_acceptable() asked whether any coding was acceptable. Nothing after that asked whether the selected coding would be applied. The "types" rule is decided after that point, and it falls back to identity. The fallback is exactly what this client refused. So no representation is left, and the answer is 406. RFC 9110 Sect. 12.5.3 makes identity acceptable only while the field does not say otherwise. An explicit "identity;q=0" says otherwise. Sect. 12.1 lets a server either honour the field or disregard it, so a server is permitted to send the identity bytes. But Unit already answers 406 to "identity;q=0" alone. The inconsistency is to answer the same request differently because a coding was named and then not applied. The 406 is not the inconsistency. The Accept-Encoding parse moves before the media type decisions. When the "types" rule withdraws the selected coding, or the response has no media type, a refused identity is answered 406. A Range does not change this: the request cannot be served at all, so the case is 406 with a Range and without one. Nothing changes here for a server with no compression configured: that exit still comes first, and the next commit is about it. A status that describes no representation is exempt. A 1xx, a 204 and a 304 have nothing to negotiate. So there is nothing that the client could refuse, and no 406 to give. The length exits at the top of the check do not cover them: nxt_http_response_content_length() stores the Content-Length field of an application without setting content_length_n, which stays -1. So an application that answered 204 or 304 with that field reached the selection and got 406. The status is tested, never the method. A HEAD carries no body, but it still describes the representation that the equivalent GET would return (RFC 9110 Sect. 9.3.2). So it is negotiated like that GET. That is why nxt_http_request_is_bodyless_final() is the wrong question here, and nxt_http_status_no_representation() is factored out beside it. On the static path, r->status is 200 at the check, before precondition evaluation. So nothing changes there, and a 406 still outranks the 304 or 412 that a validator would give. Two cases stay as they are, and now have tests. A zero-length response transfers no representation bytes, so there is nothing that the client refused: it stays 200, and a Range against it keeps its 416. A response that carries its own Content-Encoding is not identity, so the refusal does not apply to it: it stays 200 in the coding of the application. The parse now runs before the point where the check sets r->resp.vary_accept_encoding. So a 406 from the selection would leave without a Vary field, and a cache that stores error responses could give that 406 to a later client that accepts identity or gzip. nxt_http_comp_not_acceptable() sets the flag on each 406 return. nxt_http_request_error() drops the response fields, but nxt_http_request_header_send() adds Vary again from the flag. So the static 406 and the application 406 both carry "Vary: Accept-Encoding". Behaviour with a compressor configured, identity refused in each case: an applicable coding, with a Range 200 in that coding, no range below "min_length" 406 (previous commit) media type outside "types" 406 nothing acceptable at all 406 zero-length response 200 (416 with a Range) response coded by the application 200 in that coding A client that accepts identity is not affected in any of these cases: the refusal gates each new branch. Closes https://github.com/freeunitorg/freeunit/issues/390. Measured with gcc 13.3 and PHP 8.4.19: 0 warnings, ./build/tests passes, and the three suites give 100 passed, 5 skipped at this commit. Without the no-representation exemption, test_php_compression_identity_refused_no_representation fails with 406 == 204. With NXT_OK returned again where the "types" rule withdraws the coding, test_static_compression_identity_refused_without_eligible_coding and test_static_compression_406_varies_on_accept_encoding fail. Without the Vary flag on the 406, test_static_compression_406_varies_on_accept_encoding and the Vary assert in test_php_compression_identity_refused fail. Without the zero-length exit, test_static_compression_identity_refused_zero_length fails. With the Content-Encoding exit disabled, test_php_compression_identity_refused_already_coded fails. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 14 +++++ docs/changes.xml | 16 ++++++ src/nxt_http.h | 1 + src/nxt_http_compression.c | 92 ++++++++++++++++++++++++------ src/nxt_http_request.c | 24 +++++++- src/nxt_http_static.c | 4 +- test/php/comp_coded_body/index.php | 15 +++++ test/test_php_compression.py | 77 ++++++++++++++++++++++++- test/test_static_compression.py | 51 +++++++++++++++++ 9 files changed, 273 insertions(+), 21 deletions(-) create mode 100644 test/php/comp_coded_body/index.php diff --git a/CHANGES b/CHANGES index e0d84d006..a2ad831a0 100644 --- a/CHANGES +++ b/CHANGES @@ -560,6 +560,20 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 single comma-joined value RFC 9110 defines them to be; previously only the first was consulted. + *) Bugfix: a response that no compressor applies to is no longer sent + as identity to a client that refused the identity coding. With a + compressor configured, "Accept-Encoding: gzip, identity;q=0" was + accepted because gzip was acceptable. When the media type of the + response was outside "types", or the response had no media type, + the response was then sent uncompressed: the identity bytes that + the request had refused. Such a request now gets "406 Not + Acceptable", with a "Range" header and without one. A 406 from + this negotiation carries "Vary: Accept-Encoding". A 1xx, 204 or + 304 response is never negotiated, so an application that sends one + of these with a "Content-Length" is not affected. A response with + no body, and a response that carries its own "Content-Encoding", + are not affected either. + *) Bugfix: "*" in "Accept-Encoding" now stands for each enabled compressor that the client did not name. Before, it stood for the identity coding only. "identity;q=0, *;q=1" was answered "406 Not diff --git a/docs/changes.xml b/docs/changes.xml index 9c7091216..02e3f974e 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -738,6 +738,22 @@ first was consulted. + + +a response that no compressor applies to is no longer sent as identity to a +client that refused the identity coding. With a compressor configured, +"Accept-Encoding: gzip, identity;q=0" was accepted because gzip was +acceptable. When the media type of the response was outside "types", or the +response had no media type, the response was then sent uncompressed: the +identity bytes that the request had refused. Such a request now gets 406, +with a Range header and without one. A 406 from this negotiation carries +"Vary: Accept-Encoding". A 1xx, 204 or 304 response is never negotiated, so +an application that sends one of these with a Content-Length is not affected. +A response with no body, and a response that carries its own +Content-Encoding, are not affected either. + + + "*" in Accept-Encoding now stands for each enabled compressor that the client diff --git a/src/nxt_http.h b/src/nxt_http.h index b84f86d4c..484bfd9fd 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -446,6 +446,7 @@ void nxt_http_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out); nxt_bool_t nxt_http_request_is_bodyless_final(nxt_http_request_t *r, nxt_http_status_t status); +nxt_bool_t nxt_http_status_no_representation(nxt_http_status_t status); nxt_buf_t *nxt_http_buf_mem(nxt_task_t *task, nxt_http_request_t *r, size_t size); nxt_buf_t *nxt_http_buf_last(nxt_http_request_t *r); diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 880128dcb..160257ad6 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -1188,6 +1188,30 @@ nxt_http_comp_merge_vary(nxt_http_request_t *r) } +/* + * A 406 depends on Accept-Encoding as much as the representation that it + * replaces. So it carries "Vary: Accept-Encoding" too. Without it, a cache + * that stores error responses can give this 406 to a later client that could + * be served. nxt_http_request_error() drops each response field, but + * nxt_http_request_header_send() adds Vary again from this flag. + * + * Only the 406 carries Vary. When the types rule withdraws the coding, + * when the response has no media type, or when no compressor is + * configured, the 200 to a client that accepts identity has no Vary. + * A shared cache can give that 200 to a later client that refused + * identity. Vary on each such 200 is a larger change, and it is not + * made here. + */ + +static nxt_int_t +nxt_http_comp_not_acceptable(nxt_http_request_t *r) +{ + r->resp.vary_accept_encoding = 1; + + return NXT_HTTP_NOT_ACCEPTABLE; +} + + /* * Decides whether an acceptable representation exists, and remembers which * compressor would be used, without touching the response or allocating a @@ -1227,6 +1251,47 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_OK; } + /* + * A 1xx, 204 or 304 describes no representation. So there is nothing + * that the client could refuse, and no 406 to give: negotiation is + * skipped, whatever the request asked for. The length tests above do + * not cover this. An application can send Content-Length with such a + * status. A 304 carries the length of the body that the client already + * has. nxt_http_response_content_length() stores that field without + * setting content_length_n, which stays -1. + * + * r->status holds the response status at both callers. The application + * path copies it out of the response before it asks. The static path + * is at 200 here, before precondition evaluation, so a 406 still + * outranks the 304 or 412 that a validator would give. + */ + + if (nxt_http_status_no_representation(r->status)) { + return NXT_OK; + } + + if (nxt_http_comp_is_resp_content_encoded(r)) { + return NXT_OK; + } + + ret = nxt_http_comp_accept_encoding(r, &accept_encoding); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + /* + * Ask what the client accepts before anything below rules compression + * out. Before, this was asked afterwards, when the response was + * already declared serveable. That is how the own bytes of the file + * reached a client that had refused them (#390). + */ + + idx = nxt_http_comp_select_compressor(conf, r, &accept_encoding, + &identity_refused); + if (idx == -1) { + return nxt_http_comp_not_acceptable(r); + } + if (r->resp.mime_type != NULL) { mime_type = *r->resp.mime_type; } else if (r->resp.content_type != NULL) { @@ -1234,8 +1299,16 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) mime_type.length = r->resp.content_type->value_length; } + /* + * A coding was selected, but the "types" rule below can still withdraw + * it, and the fallback is always identity. When the client refused + * identity, that fallback does not exist. So the answer is 406, not + * the bytes that the client refused. "min_length" cannot reach here: a + * coding below it is not selected. + */ + if (mime_type.start == NULL) { - return NXT_OK; + return identity_refused ? nxt_http_comp_not_acceptable(r) : NXT_OK; } if (conf->mime_types_rule != NULL) { @@ -1243,14 +1316,10 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) mime_type.start, mime_type.length); if (ret == 0) { - return NXT_OK; + return identity_refused ? nxt_http_comp_not_acceptable(r) : NXT_OK; } } - if (nxt_http_comp_is_resp_content_encoded(r)) { - return NXT_OK; - } - /* * Past every early return above, so this response really was subject to * negotiation on Accept-Encoding, whichever coding is chosen below. @@ -1262,17 +1331,6 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_ERROR; } - ret = nxt_http_comp_accept_encoding(r, &accept_encoding); - if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; - } - - idx = nxt_http_comp_select_compressor(conf, r, &accept_encoding, - &identity_refused); - if (idx == -1) { - return NXT_HTTP_NOT_ACCEPTABLE; - } - ctx->sel_idx = idx; ctx->identity_refused = identity_refused; diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 9155642a1..35610f296 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -857,7 +857,8 @@ nxt_http_request_is_bodyless_final(nxt_http_request_t *r, return 0; } - if (status == NXT_HTTP_NO_CONTENT || status == NXT_HTTP_NOT_MODIFIED) { + /* 204 and 304; the 1xx arm of that test is excluded above. */ + if (nxt_http_status_no_representation(status)) { return 1; } @@ -865,6 +866,27 @@ nxt_http_request_is_bodyless_final(nxt_http_request_t *r, } +/* + * Tells whether a status describes no representation of its own: a 1xx + * interim response, a 204, and a 304. The Content-Length of a 304 describes + * the body that the client already has. Content negotiation has nothing to + * select for such a response, so the response cannot be 406. + * + * The method is not read on purpose. That separates this from + * nxt_http_request_is_bodyless_final() above. A HEAD response carries no + * body, but it still describes the representation that the equivalent GET + * would return (RFC 9110 Sect. 9.3.2). So it is negotiated like that GET. + */ + +nxt_bool_t +nxt_http_status_no_representation(nxt_http_status_t status) +{ + return status < NXT_HTTP_OK + || status == NXT_HTTP_NO_CONTENT + || status == NXT_HTTP_NOT_MODIFIED; +} + + static nxt_bool_t nxt_http_request_is_bodyless(nxt_http_request_t *r) { diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 2806c64f8..2a8f19fa3 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -960,7 +960,9 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, * satisfied. * * Not reached when nothing is acceptable: that is already 406, from - * nxt_http_comp_check_acceptable() above. + * nxt_http_comp_check_acceptable() above. That includes a response + * that no coding is applied to, where the full 200 would be + * identity again. */ if (is_range) { diff --git a/test/php/comp_coded_body/index.php b/test/php/comp_coded_body/index.php new file mode 100644 index 000000000..db8428181 --- /dev/null +++ b/test/php/comp_coded_body/index.php @@ -0,0 +1,15 @@ + 100 MB in total: more shared memory than an app may hold -def raw_get(path, encoding, timeout=30): +def raw_get(path, encoding, timeout=30, method='GET'): """ A plain socket request. The shared client cannot express "fail rather than block forever", which is exactly the failure these tests look for. @@ -153,7 +153,7 @@ def raw_get(path, encoding, timeout=30): try: sock.sendall( - f'GET {path} HTTP/1.1\r\n' + f'{method} {path} HTTP/1.1\r\n' f'Host: localhost\r\n' f'Accept-Encoding: {encoding}\r\n' f'Connection: close\r\n\r\n'.encode() @@ -294,6 +294,7 @@ def test_php_compression_identity_refused(): status, headers, _ = raw_get('/', 'identity;q=0') assert status == 406, 'no representation left for the application body' + assert headers.get('Vary') == 'Accept-Encoding', 'the 406 varies' assert 'Content-Encoding' not in headers @@ -357,3 +358,75 @@ def test_php_compression_identity_refused_below_min_length(): assert status == 200, 'identity is acceptable here' assert 'Content-Encoding' not in headers assert body == b'A' * 64, 'the body, uncompressed' + + +def test_php_compression_identity_refused_no_representation(): + # A 1xx, 204 or 304 describes no representation. So there is nothing + # that the client could refuse, and no 406 to give. + # + # The length exits at the top of nxt_http_comp_check_acceptable() do not + # cover this. nxt_http_response_content_length() stores the + # Content-Length field of an application without setting + # r->resp.content_length_n, which stays -1. So a 204 or 304 that + # carries the field reached the compressor selection and came back 406. + # The status and the framing that nxt_http_request_header_send() would + # correct never got there. + client.load('no_body_status') + configure_compression() + + status, headers, body = raw_get('/?status=204&cl', 'identity;q=0') + + assert status == 204, 'a 204 has no representation to refuse' + assert 'Content-Encoding' not in headers + assert body == b'', 'no body on a 204' + + status, headers, body = raw_get('/?status=304&cl', 'identity;q=0') + + assert status == 304, 'a 304 describes the body the client already has' + assert 'Content-Encoding' not in headers + assert body == b'', 'no body on a 304' + + # The same application, the same Content-Length, and a status that + # describes a representation: still 406. + status, headers, _ = raw_get('/?status=200&cl', 'identity;q=0') + + assert status == 406, 'a 200 still has a representation to refuse' + assert 'Content-Encoding' not in headers + + +def test_php_compression_identity_refused_head(): + # A HEAD carries no body either. But it still describes the + # representation that the equivalent GET would return (RFC 9110 + # Sect. 9.3.2). So it is negotiated like that GET, and a refused + # identity is a 406. That is why the exemption above tests the status + # only, and never the method, and why + # nxt_http_request_is_bodyless_final() is the wrong question here. + client.load('comp_large_body') + configure_compression() + + status, headers, _ = raw_get('/', 'identity;q=0', method='HEAD') + + assert status == 406, 'a HEAD is negotiated like the GET it stands for' + assert 'Content-Encoding' not in headers + + status, headers, _ = raw_get('/', 'gzip', method='HEAD') + + assert status == 200, 'the same HEAD from a client that takes gzip' + + +def test_php_compression_identity_refused_already_coded(): + # The application coded its response itself. That response is not + # identity, so the refusal of identity does not apply to it, and Unit + # does not negotiate it again: 200 in the coding of the application, + # without a compressor configured and with one. + client.load('comp_coded_body') + + for configured in (False, True): + if configured: + configure_compression() + + status, headers, body = raw_get('/', 'identity;q=0') + + assert status == 200, f'a coded response is not identity: {configured}' + assert headers.get('Content-Encoding') == 'gzip', 'the coding kept' + assert gzip.decompress(body) == b'A' * 100000, 'coded once' diff --git a/test/test_static_compression.py b/test/test_static_compression.py index 2935c8080..020ccc2f6 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -399,6 +399,57 @@ def test_static_compression_identity_refused_below_min_length(): assert 'Content-Range' not in headers +def test_static_compression_identity_refused_without_eligible_coding(): + # The configured compressor applies only to text/css. This extensionless + # file has no media type. So no coding is applied to it, and identity is + # again all that is left. This client refused identity. + for extra in ({}, {'Range': 'bytes=0-1'}): + status, headers, _ = _raw_get( + '/raw', + **{'Accept-Encoding': 'gzip, identity;q=0', **extra}, + ) + assert status == 406, 'no configured coding can serve this response' + assert 'Content-Encoding' not in headers + assert 'Content-Range' not in headers + + +def test_static_compression_identity_refused_zero_length(temp_dir): + # A zero-length response transfers no representation bytes. So there is + # nothing that the client refused, and the response is not negotiated. + # It stays 200, and a Range against it keeps its 416. + Path(f'{temp_dir}/assets/empty.css').write_bytes(b'') + + status, headers, body = _raw_get( + '/empty.css', **{'Accept-Encoding': 'gzip, identity;q=0'} + ) + + assert status == 200, 'no bytes to refuse' + assert 'Content-Encoding' not in headers + assert body == b'' + + for accept in ('gzip, identity;q=0', 'gzip'): + status, headers, _ = _raw_get( + '/empty.css', **{'Accept-Encoding': accept, 'Range': 'bytes=0-4'} + ) + + assert status == 416, f'a range against no bytes: {accept!r}' + assert headers['Content-Range'] == 'bytes */0' + + +def test_static_compression_406_varies_on_accept_encoding(): + # A 406 from negotiation depends on Accept-Encoding, like each negotiated + # response. Without "Vary: Accept-Encoding", a cache that stores error + # responses can give this 406 to a later client that accepts identity. + for url, accept in ( + ('/big.css', 'identity;q=0, *;q=0'), # nothing acceptable + ('/tiny.css', 'gzip, identity;q=0'), # gzip below "min_length" + ('/raw', 'gzip, identity;q=0'), # media type outside "types" + ): + status, headers, _ = _raw_get(url, **{'Accept-Encoding': accept}) + assert status == 406, f'{url} {accept!r}' + assert headers.get('Vary') == 'Accept-Encoding', f'{url} {accept!r}' + + def test_static_compression_min_length_is_per_compressor(): # "min_length" belongs to each compressor. When the coding with the # highest weight is below its own minimum, the next coding can still be From e62650ffc7e6a9ce9f8284f97cda4a50970adb67 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 26 Sep 2026 17:20:15 +0200 Subject: [PATCH 313/391] http: read a refused identity when no compressor is configured nxt_http_comp_check_acceptable() returned before the Accept-Encoding parse when no compression was configured. Identity is a representation that each response has. So a refusal must be read even with no compressor. Otherwise a static server with compression off gives a range of the own bytes of the file to a client that refused them: GET /index.html Accept-Encoding: identity;q=0 Range: bytes=0-4 -> 206, the first five bytes of the file That is the defect of the previous commit, seen from the other side. The "no compression" exit now comes after the parse and the selection. So "identity;q=0", and "*;q=0" with no more specific entry for identity, are 406 on each deployment. Everything else about such a server is unchanged. A zero-length file is still 200, and a Range against it is still 416: no representation bytes are transferred, so there is nothing to refuse. For that, nxt_http_comp_select_compressor() runs against a NULL configuration. Identity is recognised from the static type table with nxt_http_comp_token_is_identity(), before the enabled array is searched. nxt_http_comp_compressor_lookup_enabled() answers "unknown" for a NULL configuration. With no compressor, identity is all that the wildcard can mean, so "identity;q=0, *;q=1" is 406 there too. Cost: before this commit, a server with no compression configured never read Accept-Encoding. Now each response with a known length above zero, and a status that describes a representation, walks its own fields once for Content-Encoding and the request fields once for Accept-Encoding. When the request has the field, its value is copied into the request pool (nxt_str_cstrz()). When it has no such field, nothing is copied. A response of unknown length, and each 1xx, 204 and 304, still exit before the parse. This commit is last in the series, so that it can be dropped or reverted on its own. Without it, a client that refuses identity gets the identity bytes on a server with no compression configured, and 406 on a server with any compressor configured. Measured with gcc 13.3 and PHP 8.4.19: 0 warnings, ./build/tests passes, and the three suites give 103 passed, 5 skipped at this commit. With the "no compression" exit moved back before the parse, test_static_range_identity_refused_without_compressor, test_static_compression_wildcard_without_compressors and test_php_compression_identity_refused_without_compressor fail, and nothing else. Without the zero-length exit, the zero-length checks in test_static_range_identity_refused_without_compressor and test_static_compression_identity_refused_zero_length fail. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 10 +++++ docs/changes.xml | 13 ++++++ src/nxt_http_compression.c | 71 +++++++++++++++++++++++++-------- test/test_php_compression.py | 13 ++++++ test/test_static.py | 50 +++++++++++++++++++++++ test/test_static_compression.py | 24 +++++++++++ 6 files changed, 164 insertions(+), 17 deletions(-) diff --git a/CHANGES b/CHANGES index a2ad831a0..9ea555b29 100644 --- a/CHANGES +++ b/CHANGES @@ -593,6 +593,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 "min_length", the client gets "406 Not Acceptable" instead of the bytes it refused. + *) Change: a server with no compression configured now answers "406 Not + Acceptable" to a request whose "Accept-Encoding" refuses the identity + coding: "identity;q=0", or "*;q=0" with no more specific entry for + identity. Identity is the only coding that such a server has. + Before, it sent the response, or the requested range of it, in the + bytes that the client had refused. To read that refusal, each + response with a known length above zero now looks at its own + "Content-Encoding" and at the request "Accept-Encoding". Before, + these checks ran only with a compressor configured. + *) Bugfix: a malformed weight in "Accept-Encoding" is no longer read as a refusal. The qvalue was converted with strtod(), which takes no digits at all from "identity;q=" and from "identity;q=abc" and reports zero, diff --git a/docs/changes.xml b/docs/changes.xml index 02e3f974e..2794d5ee3 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -778,6 +778,19 @@ min_length, the client gets 406 instead of the bytes it refused. + + +a server with no compression configured now answers 406 to a request whose +Accept-Encoding refuses the identity coding: "identity;q=0", or "*;q=0" with +no more specific entry for identity. Identity is the only coding that such a +server has. Before, it sent the response, or the requested range of it, in +the bytes that the client had refused. To read that refusal, each response +with a known length above zero now looks at its own Content-Encoding and at +the request Accept-Encoding. Before, these checks ran only with a compressor +configured. + + + a malformed weight in Accept-Encoding is no longer read as a refusal. The diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 160257ad6..eb1a2fe22 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -535,6 +535,12 @@ static nxt_uint_t nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, const nxt_str_t *token) { + /* With compression off, there is no enabled array to search. */ + + if (conf == NULL) { + return NXT_HTTP_COMP_SCHEME_UNKNOWN; + } + /* * RFC 9110 Sect. 8.4.1: a content coding is a token, and tokens are * compared case-insensitively. "Identity;q=0" and "GZIP" are as valid @@ -552,6 +558,24 @@ nxt_http_comp_compressor_lookup_enabled(const nxt_http_comp_conf_t *conf, } +/* + * Tells whether the token names the identity coding. Identity is a + * representation that each response has. So it is recognised from the + * static table, not from the enabled compressors: it must be recognised + * with no compressor enabled. + */ + +static bool +nxt_http_comp_token_is_identity(const nxt_str_t *token) +{ + const nxt_http_comp_type_t *identity; + + identity = &nxt_http_comp_compressors[NXT_HTTP_COMP_SCHEME_IDENTITY]; + + return nxt_strcasestr_eq(token, &identity->token); +} + + /* * We need to parse the 'Accept-Encoding` header as described by * @@ -797,18 +821,25 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, continue; } - ecidx = nxt_http_comp_compressor_lookup_enabled(conf, &enc); - if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) { - continue; - } - - /* A built configuration holds identity at index 0. */ + /* + * Identity says whether the own bytes of the response are + * acceptable. That does not depend on the configured compressors. + * Read it before the lookup, so that a refusal still arrives when + * compression is off. That is the one case where identity is all + * the server can offer. + */ - if (ecidx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + if (nxt_http_comp_token_is_identity(&enc)) { + ecidx = NXT_HTTP_COMP_SCHEME_IDENTITY; identity_named = true; identity_named_ok = (qval != 0.0); } else { + ecidx = nxt_http_comp_compressor_lookup_enabled(conf, &enc); + if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) { + continue; + } + /* * Record the coding as named, whether or not it can be applied: * the wildcard stands only for the codings that the field did @@ -884,7 +915,7 @@ nxt_http_comp_select_compressor(const nxt_http_comp_conf_t *conf, idx = NXT_HTTP_COMP_SCHEME_IDENTITY; weight = wildcard_qval; - } else { + } else if (conf != NULL) { for (nxt_uint_t i = 1; i < conf->nr_enabled; i++) { const nxt_str_t *tok = &conf->enabled[i].type->token; @@ -1237,12 +1268,6 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1, .sel_idx = -1 }; - /* A built configuration always holds identity, so NULL is the only - "no compression" state. */ - if (conf == NULL) { - return NXT_OK; - } - if (r->resp.content_length == NULL && r->resp.content_length_n == -1) { return NXT_OK; } @@ -1281,9 +1306,11 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) /* * Ask what the client accepts before anything below rules compression - * out. Before, this was asked afterwards, when the response was - * already declared serveable. That is how the own bytes of the file - * reached a client that had refused them (#390). + * out. Identity is the one representation that the server always has. + * So "identity;q=0" is a refusal that must be answered even where no + * compressor can run. Before, this was asked afterwards, when the + * response was already declared serveable. That is how the own bytes + * of the file reached a client that had refused them (#390). */ idx = nxt_http_comp_select_compressor(conf, r, &accept_encoding, @@ -1292,6 +1319,16 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return nxt_http_comp_not_acceptable(r); } + /* + * A built configuration always holds identity. So NULL is the only "no + * compression" state, and identity is then the only coding. A client + * that refused it left through the 406 above. + */ + + if (conf == NULL) { + return NXT_OK; + } + if (r->resp.mime_type != NULL) { mime_type = *r->resp.mime_type; } else if (r->resp.content_type != NULL) { diff --git a/test/test_php_compression.py b/test/test_php_compression.py index 7f460b0d3..0487db3a6 100644 --- a/test/test_php_compression.py +++ b/test/test_php_compression.py @@ -323,6 +323,19 @@ def test_php_compression_wildcard_offers_a_coding(): assert gzip.decompress(body) == b'A' * 100000 +def test_php_compression_identity_refused_without_compressor(): + # The same request with compression off: identity is then the only + # coding, and this client refused it. This is the path that reaches the + # check with a NULL configuration. + client.load('comp_large_body') + + status, headers, _ = raw_get('/', 'identity;q=0') + + assert status == 406, 'identity is the only available coding' + assert headers.get('Vary') == 'Accept-Encoding', 'the 406 varies' + assert 'Content-Encoding' not in headers + + def test_php_compression_identity_refused_below_min_length(): # gzip is acceptable but never applied: "min_length" is above this body. # So the response would fall back to the identity that the client diff --git a/test/test_static.py b/test/test_static.py index dd584b584..b4d616c30 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -154,6 +154,56 @@ def test_static_accept_ranges(): assert resp['headers']['Accept-Ranges'] == 'bytes', 'Accept-Ranges on 200' +def test_static_range_identity_refused_without_compressor(temp_dir): + # A range is served as identity. With no compressor configured, there + # is no other representation to offer a client that refused identity. + # So the request is not serveable. Do not give it the 206 of exactly the + # bytes that it refused. Nothing here configures compression, so this is + # the path that skipped the Accept-Encoding parse before. + resp = client.get( + url='/index.html', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'identity;q=0', + 'Range': 'bytes=0-4', + }, + ) + + assert resp['status'] == 406, 'identity is the only available coding' + assert 'Content-Range' not in resp['headers'] + + # A zero-length file transfers no representation bytes. So there is + # nothing that the client refused, and the file is not negotiated: 200, + # and a Range against it keeps its 416. + Path(f'{temp_dir}/assets/empty').write_bytes(b'') + + resp = client.get( + url='/empty', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'identity;q=0', + }, + ) + + assert resp['status'] == 200, 'no bytes to refuse' + assert resp['body'] == '' + + resp = client.get( + url='/empty', + headers={ + 'Host': 'localhost', + 'Connection': 'close', + 'Accept-Encoding': 'identity;q=0', + 'Range': 'bytes=0-4', + }, + ) + + assert resp['status'] == 416, 'a range against no bytes' + assert resp['headers']['Content-Range'] == 'bytes */0' + + def unit_second(resp): """ The second Unit believes it is in, taken from the response it just sent. diff --git a/test/test_static_compression.py b/test/test_static_compression.py index 020ccc2f6..ac0bbecfc 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -592,6 +592,30 @@ def test_static_compression_wildcard_skips_inapplicable_coding(temp_dir): ).read_bytes() +def test_static_compression_wildcard_without_compressors(temp_dir): + # With no compressor enabled, identity is the only available coding. So + # identity is all that "*" can stand for, and this client refused it. + assert 'success' in client.conf_delete( + 'settings/http/compression' + ), 'compression off' + + for spelling in ( + 'identity;q=0, *;q=1', + 'identity;q=0, *;q=0.5, gzip;q=0.1', + ): + status, headers, _ = _raw_get(**{'Accept-Encoding': spelling}) + + assert status == 406, f'"*" has only identity to offer: {spelling!r}' + assert 'Content-Encoding' not in headers + + # The wildcard still accepts identity when no token named it. + status, headers, body = _raw_get(**{'Accept-Encoding': 'gzip;q=0, *;q=1'}) + + assert status == 200, 'identity is what the wildcard offers' + assert 'Content-Encoding' not in headers + assert body == Path(f'{temp_dir}/assets/big.css').read_bytes() + + def test_static_compression_range_identity_refused_guards(temp_dir): # The cases either side of it, which must not move. size = Path(f'{temp_dir}/assets/big.css').stat().st_size From 97bac4b381c41fc4d97914ed9a22c5ae5f175db4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:34:40 +0000 Subject: [PATCH 314/391] changelog: add entries for fixes merged without one Entries for #445 (shared memory record reads), the TLS configuration kept by an idle keep-alive connection, #464 (queued I/O after a close), the static file double close, the proxy's second Content-Length, #439 (request deadline at a WebSocket upgrade), #438 ("compressors" object form), #441 (the OTel parent span), #487 (a QUIT before a worker is ready, Fixes #482), #481 (a prototype that exits at shutdown) and #440 (the log level of a QUIT to an exited worker). Co-Authored-By: Claude Opus 5.5 --- CHANGES | 57 +++++++++++++++++++++++++++++ docs/changes.xml | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) diff --git a/CHANGES b/CHANGES index 9ea555b29..4b8b64bb4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,12 +1,69 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Security: the router and libunit read records from shared memory + that the other process can still write, without enough checks. A + buffer that was not a whole number of records was read past its + end. In libunit, a segment id of UINT32_MAX wrapped the size of the + segment array, a segment of any size was accepted, a duplicate + segment id replaced a segment that buffers still used, and a field + could point before the field array. The router trusted a busy chunk + marker that the application can clear. All of these are now + checked. + + *) Bugfix: a keep-alive TLS connection that was idle across a + reconfiguration could use the TLS settings of the old configuration + after they were freed, when it was closed. The connection now keeps + the configuration until it is freed. + + *) Bugfix: a read or write that was already queued when a connection + was closed could run on the closed connection: a write could call a + missing error handler, and a read could read the closing socket and + close the connection again. Such a queued handler now does + nothing. + + *) Bugfix: a static file response that failed after the request was + closed could close the file descriptor twice. The second close could + close a descriptor that another connection had got in the + meantime. + + *) Bugfix: a proxied request with a chunked body, or an HTTP/2 request + without a content-length, was sent to the upstream with two + Content-Length fields. + + *) Bugfix: with "limits": {"timeout"} set, a WebSocket session was + closed about one timeout after the upgrade, because the request + deadline stayed armed for every frame. The deadline now stops at the + upgrade. + + *) Bugfix: the router crashed when it validated a configuration that + gave "compressors" in the object form. + + *) Change: with telemetry on, the application now gets a traceparent + that names FreeUnit's own span as its parent, so the application's + spans are children of FreeUnit's span. Before, the application got + the client's traceparent unchanged. + *) Bugfix: when unitd ran as root and the control socket was given to another user or group with --control-user or --control-group, that user passed the socket file permissions but the controller then closed the connection. The controller now also accepts the configured user, and a peer in the configured group. + *) Bugfix: a worker that got a QUIT before it was ready did not act on + it, and ran until it was killed 15 seconds later, so unitd was slow + to exit. The worker now exits at once. + + *) Bugfix: at shutdown, the router could ask a prototype that was + already exiting to start a new worker. The prototype then forked a + worker that nobody stopped, and unitd did not exit. If the + prototype was gone, the router logged a false alert. The prototype + now refuses the start, and the router logs a gone prototype at the + debug level. + + *) Change: a QUIT that fails because the worker already exited is now + logged at the info level, not as an alert. + *) Security: an application could write an unexpected value into an entry of a shared memory queue. The router then retried that entry for ever. Each enqueue and dequeue now stops after one queue size diff --git a/docs/changes.xml b/docs/changes.xml index 2794d5ee3..edf8b681b 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -19,6 +19,99 @@ also accepts the configured user, and a peer in the configured group. + + +a worker that got a QUIT before it was ready did not act on it, and ran until +it was killed 15 seconds later, so unitd was slow to exit. The worker now +exits at once. + + + + + +at shutdown, the router could ask a prototype that was already exiting to +start a new worker. The prototype then forked a worker that nobody stopped, +and unitd did not exit. If the prototype was gone, the router logged a false +alert. The prototype now refuses the start, and the router logs a gone +prototype at the debug level. + + + + + +a QUIT that fails because the worker already exited is now logged at the info +level, not as an alert. + + + + + +the router and libunit read records from shared memory that the other +process can still write, without enough checks. A buffer that was not a whole +number of records was read past its end. In libunit, a segment id of +UINT32_MAX wrapped the size of the segment array, a segment of any size was +accepted, a duplicate segment id replaced a segment that buffers still used, +and a field could point before the field array. The router trusted a busy +chunk marker that the application can clear. All of these are now checked. + + + + + +a keep-alive TLS connection that was idle across a reconfiguration could +use the TLS settings of the old configuration after they were freed, when it +was closed. The connection now keeps the configuration until it is freed. + + + + + +a read or write that was already queued when a connection was closed could +run on the closed connection: a write could call a missing error handler, and +a read could read the closing socket and close the connection again. Such a +queued handler now does nothing. + + + + + +a static file response that failed after the request was closed could close +the file descriptor twice. The second close could close a descriptor that +another connection had got in the meantime. + + + + + +a proxied request with a chunked body, or an HTTP/2 request without a +content-length, was sent to the upstream with two Content-Length fields. + + + + + +with "limits": {"timeout"} set, a WebSocket session was closed about one +timeout after the upgrade, because the request deadline stayed armed for every +frame. The deadline now stops at the upgrade. + + + + + +the router crashed when it validated a configuration that gave +"compressors" in the object form. + + + + + +with telemetry on, the application now gets a traceparent that names +FreeUnit's own span as its parent, so the application's spans are children of +FreeUnit's span. Before, the application got the client's traceparent +unchanged. + + + an application could write an unexpected value into an entry of a shared From 691d455c2055bba4fa747ad9a6d96126348b7bc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:24:00 +0000 Subject: [PATCH 315/391] build(deps): Bump actions/cache from 4 to 6 Bumps [actions/cache](https://github.com/actions/cache) from 4 to 6. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/cache dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/audit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 95f50ad50..db007016a 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -146,7 +146,7 @@ jobs: # download per pin instead of 190 MB on every run and every Monday. The # digest is checked below either way, so a poisoned entry still fails. - name: Cache the Wasmtime source archive - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/wasmtime-archive key: >- From d9afbc698b3a8eb4426fd930d809ee9a2f6a98cf Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 16:38:13 +0000 Subject: [PATCH 316/391] test(fuzz): add a libFuzzer target for the chunked body decoder nxt_http_chunk_parse() decodes a chunked request body. In the proxy, it also decodes a chunked upstream response body. No fuzz target reached it. fuzz_http_chunk decodes each input two times. One time as one buffer. One time cut into pieces of 1 to 16 bytes, which it gives to the decoder as chains of one to three buffers for each call. The first input byte seeds the cuts. The two runs must give the same verdict, the same position of the byte that ended or broke the message, and the same decoded bytes. Each decoded slice must be inside the buffer it came from. A difference causes an abort. Thus a state that is lost at a buffer border is reported like a crash. Each piece is a separate heap allocation of its exact size. Thus a read after the end of a piece hits an ASan redzone. The target destroys the pool after each input. The seed corpus comes from the chunk parse unit test and from the proxy chunked tests: trailers, chunk extensions, a pipelined request after the last chunk, a trailer line without a colon, a size overflow and a short body. Co-Authored-By: Claude Opus 5.5 --- auto/fuzzing | 10 + auto/sources | 1 + .../hex_lower_ext.bin | 4 + .../fuzz_http_chunk_seed_corpus/hex_upper.bin | 4 + .../incomplete.bin | 2 + .../last_chunk.bin | Bin 0 -> 6 bytes .../fuzz_http_chunk_seed_corpus/long_data.bin | 6 + .../fuzz_http_chunk_seed_corpus/pipelined.bin | 7 + .../short_data.bin | 4 + .../fuzz_http_chunk_seed_corpus/size_max.bin | 2 + .../size_overflow.bin | 2 + .../smuggle_colonless.bin | 3 + .../fuzz_http_chunk_seed_corpus/trailer.bin | 5 + .../trailer_obs_text.bin | 3 + .../trailers_two.bin | 6 + .../two_chunks.bin | 6 + fuzzing/nxt_http_chunk_fuzz.c | 328 ++++++++++++++++++ 17 files changed, 393 insertions(+) create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/hex_lower_ext.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/hex_upper.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/incomplete.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/last_chunk.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/long_data.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/pipelined.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/short_data.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/size_max.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/size_overflow.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/smuggle_colonless.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/trailer.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/trailer_obs_text.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/trailers_two.bin create mode 100644 fuzzing/fuzz_http_chunk_seed_corpus/two_chunks.bin create mode 100644 fuzzing/nxt_http_chunk_fuzz.c diff --git a/auto/fuzzing b/auto/fuzzing index 590ca7948..155a4e0ca 100644 --- a/auto/fuzzing +++ b/auto/fuzzing @@ -20,6 +20,7 @@ if [ -n "$NXT_FUZZ" ]; then .PHONY: fuzz fuzz: $NXT_BUILD_DIR/fuzz_basic \\ + $NXT_BUILD_DIR/fuzz_http_chunk \\ $NXT_BUILD_DIR/fuzz_http_controller \\ $NXT_BUILD_DIR/fuzz_http_h1p \\ $NXT_BUILD_DIR/fuzz_http_h1p_peer \\ @@ -36,6 +37,15 @@ $NXT_BUILD_DIR/fuzz_basic: \$(NXT_FUZZ_OBJS) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ $NXT_FUZZ +$NXT_BUILD_DIR/fuzz_http_chunk: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_http_chunk \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_http_chunk_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + $NXT_BUILD_DIR/fuzz_http_controller: \$(NXT_FUZZ_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(PP_LD) \$@ diff --git a/auto/sources b/auto/sources index 214455678..d076af702 100644 --- a/auto/sources +++ b/auto/sources @@ -359,6 +359,7 @@ fi NXT_FUZZ_SRCS=" \ fuzzing/nxt_basic_fuzz.c \ + fuzzing/nxt_http_chunk_fuzz.c \ fuzzing/nxt_http_controller_fuzz.c \ fuzzing/nxt_http_h1p_fuzz.c \ fuzzing/nxt_http_h1p_peer_fuzz.c \ diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/hex_lower_ext.bin b/fuzzing/fuzz_http_chunk_seed_corpus/hex_lower_ext.bin new file mode 100644 index 000000000..8f4b6195f --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/hex_lower_ext.bin @@ -0,0 +1,4 @@ +oa;name=value +0123456789 +0;x + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/hex_upper.bin b/fuzzing/fuzz_http_chunk_seed_corpus/hex_upper.bin new file mode 100644 index 000000000..311021e7c --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/hex_upper.bin @@ -0,0 +1,4 @@ +JA +0123456789 +0 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/incomplete.bin b/fuzzing/fuzz_http_chunk_seed_corpus/incomplete.bin new file mode 100644 index 000000000..8ce123b02 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/incomplete.bin @@ -0,0 +1,2 @@ +á65 +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \ No newline at end of file diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/last_chunk.bin b/fuzzing/fuzz_http_chunk_seed_corpus/last_chunk.bin new file mode 100644 index 0000000000000000000000000000000000000000..cd96a933a5575c11035644424d7a2663a62160e9 GIT binary patch literal 6 NcmZQD;N{}w0ssMQ0ABzA literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/long_data.bin b/fuzzing/fuzz_http_chunk_seed_corpus/long_data.bin new file mode 100644 index 000000000..c5620c711 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/long_data.bin @@ -0,0 +1,6 @@ +¼40 +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +40 +BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB +0 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/pipelined.bin b/fuzzing/fuzz_http_chunk_seed_corpus/pipelined.bin new file mode 100644 index 000000000..bf8dedfc1 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/pipelined.bin @@ -0,0 +1,7 @@ +5 +hello +0 + +GET / HTTP/1.1 +Host: localhost + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/short_data.bin b/fuzzing/fuzz_http_chunk_seed_corpus/short_data.bin new file mode 100644 index 000000000..401263434 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/short_data.bin @@ -0,0 +1,4 @@ +—2 +X +0 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/size_max.bin b/fuzzing/fuzz_http_chunk_seed_corpus/size_max.bin new file mode 100644 index 000000000..f151b4124 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/size_max.bin @@ -0,0 +1,2 @@ +rfffffffffffffff +AAAA \ No newline at end of file diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/size_overflow.bin b/fuzzing/fuzz_http_chunk_seed_corpus/size_overflow.bin new file mode 100644 index 000000000..db8379ef9 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/size_overflow.bin @@ -0,0 +1,2 @@ +M10000000000000000 +A diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/smuggle_colonless.bin b/fuzzing/fuzz_http_chunk_seed_corpus/smuggle_colonless.bin new file mode 100644 index 000000000..4da71a974 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/smuggle_colonless.bin @@ -0,0 +1,3 @@ +(0 +GET /admin HTTP/1.1 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/trailer.bin b/fuzzing/fuzz_http_chunk_seed_corpus/trailer.bin new file mode 100644 index 000000000..6393620b7 --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/trailer.bin @@ -0,0 +1,5 @@ +”3 +abc +0 +X-Trailer-Test: trailed + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/trailer_obs_text.bin b/fuzzing/fuzz_http_chunk_seed_corpus/trailer_obs_text.bin new file mode 100644 index 000000000..936b3910c --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/trailer_obs_text.bin @@ -0,0 +1,3 @@ +Þ0 +X: € + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/trailers_two.bin b/fuzzing/fuzz_http_chunk_seed_corpus/trailers_two.bin new file mode 100644 index 000000000..ca22f01da --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/trailers_two.bin @@ -0,0 +1,6 @@ +¹1 +z +0 +A: 1 +B: 2 + diff --git a/fuzzing/fuzz_http_chunk_seed_corpus/two_chunks.bin b/fuzzing/fuzz_http_chunk_seed_corpus/two_chunks.bin new file mode 100644 index 000000000..dc568458a --- /dev/null +++ b/fuzzing/fuzz_http_chunk_seed_corpus/two_chunks.bin @@ -0,0 +1,6 @@ +%2 +ab +2 +cd +0 + diff --git a/fuzzing/nxt_http_chunk_fuzz.c b/fuzzing/nxt_http_chunk_fuzz.c new file mode 100644 index 000000000..6010d9f19 --- /dev/null +++ b/fuzzing/nxt_http_chunk_fuzz.c @@ -0,0 +1,328 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +#include + + +/* + * This target fuzzes the chunked transfer-coding decoder, + * nxt_http_chunk_parse(). The decoder reads a client request body and, in + * the proxy, an upstream response body. Thus both ends of a connection + * send data to it. + * + * The decoder keeps its state between buffers and between calls, and a + * network read can stop at any byte. Thus the target decodes each input + * two times: one time as a single buffer, and one time cut into pieces that + * it gives to the decoder as chains of one to three buffers for each call. + * The two runs must give the same verdict, the same end position of the + * message, and the same decoded bytes. A difference causes an abort. Thus + * a state that is lost or read incorrectly at a buffer border is a crash, + * not a silent difference. + * + * Input layout: the first byte seeds the cut points, the other bytes are the + * body. + */ + + +#define KMININPUTLENGTH 2 +#define KMAXINPUTLENGTH 4096 + +#define NXT_CHUNK_FUZZ_MAX_PIECES 64 + + +typedef struct { + nxt_uint_t verdict; + size_t consumed; + size_t length; + size_t capacity; + u_char *data; +} nxt_chunk_fuzz_result_t; + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +static nxt_int_t nxt_chunk_fuzz_run(nxt_task_t *task, nxt_mp_t *mp, + const u_char *body, size_t size, uint32_t seed, + nxt_chunk_fuzz_result_t *res); +static nxt_int_t nxt_chunk_fuzz_collect(nxt_buf_t *out, + nxt_chunk_fuzz_result_t *res); +static nxt_uint_t nxt_chunk_fuzz_verdict(nxt_http_chunk_parse_t *hcp); +static uint32_t nxt_chunk_fuzz_rand(uint32_t *state); + + +extern char **environ; + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + if (nxt_lib_start("fuzzing", NULL, &environ) != NXT_OK) { + return NXT_ERROR; + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + nxt_mp_t *mp; + nxt_task_t *task; + nxt_thread_t *thr; + nxt_chunk_fuzz_result_t whole, split; + + if (size < KMININPUTLENGTH || size > KMAXINPUTLENGTH) { + return 0; + } + + thr = nxt_thread(); + task = thr->task; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return 0; + } + + /* A seed of 0 puts the full body in one buffer: the reference run. */ + + if (nxt_chunk_fuzz_run(task, mp, data + 1, size - 1, 0, &whole) != NXT_OK + || nxt_chunk_fuzz_run(task, mp, data + 1, size - 1, + (uint32_t) data[0] + 1, &split) + != NXT_OK) + { + goto done; + } + + if (whole.verdict != split.verdict + || whole.consumed != split.consumed + || whole.length != split.length + || memcmp(whole.data, split.data, whole.length) != 0) + { + nxt_thread_log_alert("chunk fuzz: split run differs: " + "verdict %ui/%ui consumed %uz/%uz " + "length %uz/%uz", + whole.verdict, split.verdict, + whole.consumed, split.consumed, + whole.length, split.length); + abort(); + } + +done: + + nxt_mp_destroy(mp); + + return 0; +} + + +/* + * Each piece is a separate heap allocation of its exact size. Thus a read + * after the end of a buffer goes into an ASan redzone, not into the next + * piece. The buffer headers and the output buffers of the decoder come from + * "mp". The caller destroys the pool after the two runs. This also frees + * the output buffers that the decoder kept in the pool. + */ + +static nxt_int_t +nxt_chunk_fuzz_run(nxt_task_t *task, nxt_mp_t *mp, const u_char *body, + size_t size, uint32_t seed, nxt_chunk_fuzz_result_t *res) +{ + size_t offset, len, cut; + size_t offsets[NXT_CHUNK_FUZZ_MAX_PIECES]; + nxt_int_t ret; + nxt_buf_t *b, *out, *pieces[NXT_CHUNK_FUZZ_MAX_PIECES]; + nxt_uint_t i, j, n, group; + nxt_http_chunk_parse_t hcp; + + nxt_memzero(res, sizeof(nxt_chunk_fuzz_result_t)); + + res->data = nxt_mp_nget(mp, size); + if (res->data == NULL) { + return NXT_ERROR; + } + + res->capacity = size; + + ret = NXT_ERROR; + + /* Cut the body. */ + + n = 0; + offset = 0; + + while (offset < size) { + len = size - offset; + + if (seed != 0 && n < NXT_CHUNK_FUZZ_MAX_PIECES - 1) { + cut = nxt_chunk_fuzz_rand(&seed) % 16 + 1; + len = nxt_min(len, cut); + } + + b = nxt_buf_mem_alloc(mp, 0, 0); + if (b == NULL) { + goto done; + } + + b->mem.start = nxt_malloc(len); + if (b->mem.start == NULL) { + goto done; + } + + nxt_memcpy(b->mem.start, body + offset, len); + + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start + len; + b->mem.end = b->mem.free; + + offsets[n] = offset; + pieces[n++] = b; + + offset += len; + } + + /* Decode. */ + + nxt_memzero(&hcp, sizeof(nxt_http_chunk_parse_t)); + + hcp.mem_pool = mp; + + /* + * Without this flag, the decoder puts a buffer that gave no data on the + * work queue of the engine. There is no engine here. + */ + hcp.retain_buffers = 1; + + for (i = 0; i < n; i += group) { + group = 1; + + if (seed != 0) { + cut = nxt_chunk_fuzz_rand(&seed) % 3 + 1; + group = nxt_min(n - i, cut); + } + + for (j = i; j < i + group - 1; j++) { + pieces[j]->next = pieces[j + 1]; + } + + pieces[i + group - 1]->next = NULL; + + out = nxt_http_chunk_parse(task, &hcp, pieces[i]); + + if (nxt_chunk_fuzz_collect(out, res) != NXT_OK) { + nxt_thread_log_alert("chunk fuzz: bad output buffer"); + abort(); + } + + if (hcp.error) { + /* No memory. This is not a verdict on the input. */ + goto done; + } + + if (hcp.chunk_error || hcp.last) { + break; + } + } + + res->verdict = nxt_chunk_fuzz_verdict(&hcp); + + /* + * The decoder stops in the buffer that holds the byte that ended the + * message or broke it. It has consumed that byte. The decoder does not + * touch the buffers after it. A buffer before it can keep "pos" at + * "start" when it held only chunk data. Thus the last piece with a + * moved "pos" is the one that holds the byte. + */ + if (hcp.chunk_error || hcp.last) { + for (j = n; j > 0; j--) { + b = pieces[j - 1]; + + if (b->mem.pos != b->mem.start) { + res->consumed = offsets[j - 1] + (b->mem.pos - b->mem.start); + break; + } + } + } + + ret = NXT_OK; + +done: + + for (i = 0; i < n; i++) { + nxt_free(pieces[i]->mem.start); + } + + return ret; +} + + +/* + * A decoded slice must be inside the buffer it was cut from, which is its + * parent. All the slices together cannot hold more bytes than the body. + */ + +static nxt_int_t +nxt_chunk_fuzz_collect(nxt_buf_t *out, nxt_chunk_fuzz_result_t *res) +{ + size_t len; + nxt_buf_t *b, *parent; + + for (b = out; b != NULL; b = b->next) { + parent = b->parent; + + if (parent == NULL + || b->mem.pos > b->mem.free + || b->mem.pos < parent->mem.start + || b->mem.free > parent->mem.end) + { + return NXT_ERROR; + } + + len = b->mem.free - b->mem.pos; + + if (len > res->capacity - res->length) { + return NXT_ERROR; + } + + nxt_memcpy(res->data + res->length, b->mem.pos, len); + res->length += len; + } + + return NXT_OK; +} + + +static nxt_uint_t +nxt_chunk_fuzz_verdict(nxt_http_chunk_parse_t *hcp) +{ + if (hcp->chunk_error) { + return 1; + } + + if (hcp->last) { + return 2; + } + + return 0; +} + + +/* xorshift32: fast, and the same sequence for the same input. */ + +static uint32_t +nxt_chunk_fuzz_rand(uint32_t *state) +{ + uint32_t x; + + x = *state; + + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + + *state = x; + + return x; +} From 2591e5a5973d47b529e7653fd79756d649d2073d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 16:39:09 +0000 Subject: [PATCH 317/391] test(fuzz): add a libFuzzer target for the upstream response header The proxy reads an upstream response header with nxt_h1p_peer_header_parse(): a status line, then header fields through nxt_http_parse_fields(). fuzz_http_h1p_peer runs the peer field handlers, but it gets to them through the request parser and in one call. No fuzz target reached the status line or the continuation after NXT_AGAIN. fuzz_http_h1p_peer_response parses each input two times. One time the full response is in the buffer at the first call. One time the response comes 1 to 32 bytes at a time. Each pass copies the next bytes to bm.free, in the same way as a read from the upstream, and the parser continues after each NXT_AGAIN, as nxt_h1p_peer_header_read_done() does. The copy is necessary: after a 1xx the parser moves the unread bytes to the start of the buffer and moves bm.free back, so bm.free is not a cursor into the input. The first input byte seeds the read sizes. The two runs must give the same result, the same status, the same header size and the same fields (hash, name and value offsets and lengths). For an error, only the outcome must be the same: a field name cut after its length limit is "too large", the same name with a bad byte after it is "invalid". Each field must be inside the header. After a complete header, the peer field handlers run, and then the framing checks of nxt_h1p_peer_header_read_done(): a response without a body ends at the empty line, a Transfer-Encoding the proxy cannot decode gives 502, and chunked together with Content-Length gives 502. The seed corpus covers Content-Length, chunked, "Chunked", "gzip, chunked", chunked and Content-Length together, duplicate and invalid lengths, HTTP/1.0, 1xx, 204 and 304, bare LF line ends and more than 16 fields. Co-Authored-By: Claude Opus 5.5 --- auto/fuzzing | 10 + auto/sources | 1 + .../bad_length.bin | 3 + .../bad_version.bin | 2 + .../chunked_and_length.bin | 4 + .../dup_length.bin | 5 + .../http10_close.bin | 4 + .../interim.bin | 5 + .../lf_only.bin | 4 + .../many_fields.bin | 22 ++ .../no_content.bin | 3 + .../not_modified.bin | 3 + .../ok_chunked.bin | 8 + .../ok_length.bin | Bin 0 -> 62 bytes .../short_status.bin | 1 + .../te_gzip_chunked.bin | 5 + .../te_upper.bin | 5 + .../underscore.bin | 4 + fuzzing/nxt_http_h1p_peer_response_fuzz.c | 358 ++++++++++++++++++ 19 files changed, 447 insertions(+) create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_length.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_version.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/chunked_and_length.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/dup_length.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/http10_close.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/interim.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/lf_only.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/many_fields.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/no_content.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/not_modified.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_chunked.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_length.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/short_status.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_gzip_chunked.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_upper.bin create mode 100644 fuzzing/fuzz_http_h1p_peer_response_seed_corpus/underscore.bin create mode 100644 fuzzing/nxt_http_h1p_peer_response_fuzz.c diff --git a/auto/fuzzing b/auto/fuzzing index 155a4e0ca..2d8499d40 100644 --- a/auto/fuzzing +++ b/auto/fuzzing @@ -24,6 +24,7 @@ fuzz: $NXT_BUILD_DIR/fuzz_basic \\ $NXT_BUILD_DIR/fuzz_http_controller \\ $NXT_BUILD_DIR/fuzz_http_h1p \\ $NXT_BUILD_DIR/fuzz_http_h1p_peer \\ + $NXT_BUILD_DIR/fuzz_http_h1p_peer_response \\ $NXT_BUILD_DIR/fuzz_json \\ $NXT_BUILD_DIR/fuzz_router_app_response \\ $NXT_BUILD_DIR/fuzz_unit_msg @@ -73,6 +74,15 @@ $NXT_BUILD_DIR/fuzz_http_h1p_peer: \$(NXT_FUZZ_OBJS) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ $NXT_FUZZ +$NXT_BUILD_DIR/fuzz_http_h1p_peer_response: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_http_h1p_peer_response \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_http_h1p_peer_response_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + $NXT_BUILD_DIR/fuzz_json: \$(NXT_FUZZ_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(PP_LD) \$@ diff --git a/auto/sources b/auto/sources index d076af702..a10bab80c 100644 --- a/auto/sources +++ b/auto/sources @@ -363,6 +363,7 @@ NXT_FUZZ_SRCS=" \ fuzzing/nxt_http_controller_fuzz.c \ fuzzing/nxt_http_h1p_fuzz.c \ fuzzing/nxt_http_h1p_peer_fuzz.c \ + fuzzing/nxt_http_h1p_peer_response_fuzz.c \ fuzzing/nxt_json_fuzz.c \ fuzzing/nxt_router_app_response_fuzz.c \ fuzzing/nxt_unit_msg_fuzz.c \ diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_length.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_length.bin new file mode 100644 index 000000000..16e2a1cfb --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_length.bin @@ -0,0 +1,3 @@ +ÔHTTP/1.1 200 OK +Content-Length: 99999999999999999999999 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_version.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_version.bin new file mode 100644 index 000000000..1851c2bb1 --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/bad_version.bin @@ -0,0 +1,2 @@ +|HTTP/2.0 200 OK + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/chunked_and_length.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/chunked_and_length.bin new file mode 100644 index 000000000..2baf93fbf --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/chunked_and_length.bin @@ -0,0 +1,4 @@ +jHTTP/1.1 200 OK +Transfer-Encoding: chunked +Content-Length: 5 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/dup_length.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/dup_length.bin new file mode 100644 index 000000000..c31c721ca --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/dup_length.bin @@ -0,0 +1,5 @@ +ŸHTTP/1.1 200 OK +Content-Length: 5 +Content-Length: 6 + +hello \ No newline at end of file diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/http10_close.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/http10_close.bin new file mode 100644 index 000000000..398fbc51f --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/http10_close.bin @@ -0,0 +1,4 @@ + HTTP/1.0 200 OK +Connection: close +X-Custom: value + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/interim.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/interim.bin new file mode 100644 index 000000000..11fe728ac --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/interim.bin @@ -0,0 +1,5 @@ +¨HTTP/1.1 100 Continue + +HTTP/1.1 200 OK +Content-Length: 0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/lf_only.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/lf_only.bin new file mode 100644 index 000000000..77f5df5ab --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/lf_only.bin @@ -0,0 +1,4 @@ +ÝHTTP/1.1 502 Bad Gateway +Content-Type: text/plain +X-A: b + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/many_fields.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/many_fields.bin new file mode 100644 index 000000000..78a340c3e --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/many_fields.bin @@ -0,0 +1,22 @@ +HTTP/1.1 200 OK +X-Field-0: v0 +X-Field-1: v1 +X-Field-2: v2 +X-Field-3: v3 +X-Field-4: v4 +X-Field-5: v5 +X-Field-6: v6 +X-Field-7: v7 +X-Field-8: v8 +X-Field-9: v9 +X-Field-10: v10 +X-Field-11: v11 +X-Field-12: v12 +X-Field-13: v13 +X-Field-14: v14 +X-Field-15: v15 +X-Field-16: v16 +X-Field-17: v17 +X-Field-18: v18 +X-Field-19: v19 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/no_content.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/no_content.bin new file mode 100644 index 000000000..0aeae015c --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/no_content.bin @@ -0,0 +1,3 @@ +>HTTP/1.1 204 No Content +Content-Length: 10 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/not_modified.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/not_modified.bin new file mode 100644 index 000000000..a30a9c427 --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/not_modified.bin @@ -0,0 +1,3 @@ +sHTTP/1.1 304 Not Modified +Transfer-Encoding: chunked + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_chunked.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_chunked.bin new file mode 100644 index 000000000..8af2544e2 --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_chunked.bin @@ -0,0 +1,8 @@ +5HTTP/1.1 200 OK +Transfer-Encoding: chunked +Date: Thu, 01 Jan 1970 00:00:00 GMT + +2 +ab +0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_length.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/ok_length.bin new file mode 100644 index 0000000000000000000000000000000000000000..54c5b5208dff5f083267018455385fb0ee7e2ace GIT binary patch literal 62 zcmZSZ2nh+$H`Fs!FfuSu@b~8Ba?Z~yNzE(K^-0Z3FUhb{Fy-Y6PAw`+EwWN5EhsK2 RN=?k=<>KYaNX^N~2LKPj5*Gjf literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/short_status.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/short_status.bin new file mode 100644 index 000000000..2aeb892ec --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/short_status.bin @@ -0,0 +1 @@ +±HTTP/1.1 20 \ No newline at end of file diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_gzip_chunked.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_gzip_chunked.bin new file mode 100644 index 000000000..691fa799d --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_gzip_chunked.bin @@ -0,0 +1,5 @@ +9HTTP/1.1 200 OK +Transfer-Encoding: gzip, chunked + +0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_upper.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_upper.bin new file mode 100644 index 000000000..88709a92f --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/te_upper.bin @@ -0,0 +1,5 @@ +7HTTP/1.1 200 OK +Transfer-Encoding: Chunked + +0 + diff --git a/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/underscore.bin b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/underscore.bin new file mode 100644 index 000000000..759802f1d --- /dev/null +++ b/fuzzing/fuzz_http_h1p_peer_response_seed_corpus/underscore.bin @@ -0,0 +1,4 @@ +GHTTP/1.1 200 OK +X_Under: 1 +Set-Cookie: a=b + diff --git a/fuzzing/nxt_http_h1p_peer_response_fuzz.c b/fuzzing/nxt_http_h1p_peer_response_fuzz.c new file mode 100644 index 000000000..79eaced9b --- /dev/null +++ b/fuzzing/nxt_http_h1p_peer_response_fuzz.c @@ -0,0 +1,358 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +#include + +/* DO NOT TRY THIS AT HOME! */ +#include "nxt_h1proto.c" + + +/* + * This target fuzzes how the proxy reads an upstream response header: the + * status line and the header fields that nxt_h1p_peer_header_parse() takes + * from the upstream socket, and the peer field handlers that run on them. + * fuzz_http_h1p_peer gets to those handlers through the request parser and + * in one call. This target gets to them in the same way as the proxy. + * + * The proxy reads the header into one buffer. After each read that + * returned NXT_AGAIN, it parses again, and the parser continues from its + * saved state. Thus the target parses each input two times: one time + * complete, and one time a few bytes at a time. The two runs must give the + * same result, the same status, the same end of the header and the same + * fields. A difference causes an abort. + * + * Input layout: the first byte seeds the read sizes, the other bytes are + * the response. + */ + + +#define KMININPUTLENGTH 2 +#define KMAXINPUTLENGTH 4096 + + +typedef struct { + nxt_int_t ret; + nxt_int_t status; + size_t header_size; + nxt_uint_t nfields; + nxt_http_field_t fields[64]; +} nxt_peer_fuzz_result_t; + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +static nxt_int_t nxt_peer_fuzz_run(nxt_mp_t *mp, u_char *start, + const u_char *src, size_t size, uint32_t seed, nxt_peer_fuzz_result_t *res); +static nxt_int_t nxt_peer_fuzz_result(nxt_int_t ret); +static nxt_bool_t nxt_peer_fuzz_field_eq(nxt_http_field_t *f1, u_char *base1, + nxt_http_field_t *f2, u_char *base2); +static uint32_t nxt_peer_fuzz_rand(uint32_t *state); + + +extern char **environ; + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + nxt_int_t ret; + + if (nxt_lib_start("fuzzing", NULL, &environ) != NXT_OK) { + return NXT_ERROR; + } + + /* Keep a fuzzing run quiet: nothing below alert is worth printing. */ + nxt_main_log.level = NXT_LOG_ALERT; + + ret = nxt_http_fields_hash(&nxt_h1p_peer_fields_hash, + nxt_h1p_peer_fields, + nxt_nitems(nxt_h1p_peer_fields)); + if (ret != NXT_OK) { + return NXT_ERROR; + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + u_char *whole_buf, *split_buf; + nxt_mp_t *mp; + nxt_uint_t i; + nxt_peer_fuzz_result_t *whole, *split; + + if (size < KMININPUTLENGTH || size > KMAXINPUTLENGTH) { + return 0; + } + + size--; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return 0; + } + + /* + * Each run gets its own heap buffer of the exact size. Thus ASan catches + * a read after the received bytes at the end of the input at least, and + * the addresses of the fields show which run they came from. + */ + whole_buf = nxt_malloc(size); + split_buf = nxt_malloc(size); + whole = nxt_mp_zget(mp, sizeof(nxt_peer_fuzz_result_t)); + split = nxt_mp_zget(mp, sizeof(nxt_peer_fuzz_result_t)); + + if (whole_buf == NULL || split_buf == NULL + || whole == NULL || split == NULL) + { + goto done; + } + + /* A seed of 0 gives the full response in one pass: the reference run. */ + + if (nxt_peer_fuzz_run(mp, whole_buf, data + 1, size, 0, whole) != NXT_OK + || nxt_peer_fuzz_run(mp, split_buf, data + 1, size, + (uint32_t) data[0] + 1, split) + != NXT_OK) + { + goto done; + } + + if (whole->ret != split->ret) { + goto differs; + } + + if (whole->ret != NXT_DONE) { + goto done; + } + + if (whole->status != split->status + || whole->header_size != split->header_size + || whole->nfields != split->nfields) + { + goto differs; + } + + for (i = 0; i < whole->nfields; i++) { + if (!nxt_peer_fuzz_field_eq(&whole->fields[i], whole_buf, + &split->fields[i], split_buf)) + { + goto differs; + } + } + +done: + + nxt_free(whole_buf); + nxt_free(split_buf); + + nxt_mp_destroy(mp); + + return 0; + +differs: + + nxt_thread_log_alert("peer response fuzz: split run differs: " + "ret %i/%i status %i/%i header %uz/%uz " + "fields %ui/%ui", + whole->ret, split->ret, + whole->status, split->status, + whole->header_size, split->header_size, + whole->nfields, split->nfields); + abort(); +} + + +static nxt_int_t +nxt_peer_fuzz_run(nxt_mp_t *mp, u_char *start, const u_char *src, size_t size, + uint32_t seed, nxt_peer_fuzz_result_t *res) +{ + size_t len, cut, done; + nxt_int_t ret; + nxt_buf_mem_t bm; + nxt_h1proto_t *h1p; + nxt_http_field_t *f; + nxt_http_peer_t *peer; + nxt_http_request_t *r; + + r = nxt_mp_zget(mp, sizeof(nxt_http_request_t)); + peer = nxt_mp_zget(mp, sizeof(nxt_http_peer_t)); + h1p = nxt_mp_zget(mp, sizeof(nxt_h1proto_t)); + + if (r == NULL || peer == NULL || h1p == NULL) { + return NXT_ERROR; + } + + /* + * A field handler can log, and nxt_log() reads task->log. The request + * is zeroed memory here. Without this line, a handler that logs causes + * a null dereference in the harness, which is not a finding. + */ + r->task.log = &nxt_main_log; + r->mem_pool = mp; + r->peer = peer; + r->resp.content_length_n = -1; + + peer->request = r; + peer->proto.h1 = h1p; + + /* The same values that nxt_h1p_peer_header_send() leaves. */ + peer->status = NXT_HTTP_UNSET; + + ret = nxt_http_parse_request_init(&h1p->parser, mp); + if (ret != NXT_OK) { + return NXT_ERROR; + } + + bm.start = start; + bm.pos = start; + bm.free = start; + bm.end = start + size; + + /* + * Each pass copies the next bytes of the input to bm.free, in the same + * way as a read from the upstream. After a 1xx, the parser can move the + * unread bytes to bm.start and move bm.free back. Thus bm.free is not a + * cursor into the input. "done" is the cursor. + */ + done = 0; + + for ( ;; ) { + len = size - done; + + if (seed != 0) { + cut = nxt_peer_fuzz_rand(&seed) % 32 + 1; + len = nxt_min(len, cut); + } + + nxt_memcpy(bm.free, src + done, len); + bm.free += len; + done += len; + + ret = nxt_h1p_peer_header_parse(peer, &bm); + + if (ret != NXT_AGAIN || done == size) { + break; + } + } + + if (bm.pos < start || bm.pos > bm.free) { + nxt_thread_log_alert("peer response fuzz: pos out of the buffer"); + abort(); + } + + res->ret = nxt_peer_fuzz_result(ret); + + if (ret != NXT_DONE) { + return NXT_OK; + } + + res->status = peer->status; + res->header_size = bm.pos - start; + + /* The same steps that nxt_h1p_peer_header_read_done() does. */ + + peer->num_inline_fields = h1p->parser.num_inline_fields; + if (peer->num_inline_fields > 0) { + nxt_memcpy(peer->inline_fields, h1p->parser.inline_fields, + sizeof(nxt_http_field_t) * peer->num_inline_fields); + } + peer->fields = h1p->parser.fields; + + nxt_http_fields_each(f, peer->inline_fields, peer->num_inline_fields, + peer->fields) + { + if (f->name < start || f->name + f->name_length > bm.pos + || f->value < start || f->value + f->value_length > bm.pos) + { + nxt_thread_log_alert("peer response fuzz: field out of header"); + abort(); + } + + if (res->nfields < nxt_nitems(res->fields)) { + res->fields[res->nfields++] = *f; + } + + } nxt_http_fields_loop; + + ret = nxt_http_fields_process(peer->inline_fields, peer->num_inline_fields, + peer->fields, &nxt_h1p_peer_fields_hash, r); + if (ret != NXT_OK) { + return NXT_ERROR; + } + + /* + * The framing decision of nxt_h1p_peer_header_read_done(), in the same + * order: a response without a body ends at the empty line, whatever the + * fields say. After that, a Transfer-Encoding the proxy cannot decode, + * and chunked together with Content-Length, each end in 502. + */ + + h1p->chunked = (h1p->transfer_encoding == NXT_HTTP_TE_CHUNKED); + + if (nxt_http_request_is_bodyless_final(r, peer->status)) { + h1p->chunked = 0; + + } else if (h1p->transfer_encoding == NXT_HTTP_TE_UNSUPPORTED + || (h1p->chunked && r->resp.content_length != NULL)) + { + res->status = NXT_HTTP_BAD_GATEWAY; + } + + return NXT_OK; +} + + +/* + * The error that a bad header ends in can depend on where it was cut. A + * field name that is longer than its limit is "too large" when the parser + * stops in it, but "invalid" when the bad byte after it is already there. + * Only the outcome must be the same. + */ + +static nxt_int_t +nxt_peer_fuzz_result(nxt_int_t ret) +{ + switch (ret) { + case NXT_DONE: + case NXT_AGAIN: + return ret; + default: + return NXT_ERROR; + } +} + + +static nxt_bool_t +nxt_peer_fuzz_field_eq(nxt_http_field_t *f1, u_char *base1, + nxt_http_field_t *f2, u_char *base2) +{ + return f1->hash == f2->hash + && f1->name_length == f2->name_length + && f1->value_length == f2->value_length + && f1->name - base1 == f2->name - base2 + && f1->value - base1 == f2->value - base2; +} + + +/* xorshift32: fast, and the same sequence for the same input. */ + +static uint32_t +nxt_peer_fuzz_rand(uint32_t *state) +{ + uint32_t x; + + x = *state; + + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + + *state = x; + + return x; +} From e89b9263277a5c44b3863fcec1ff95e80cd7d0a0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 16:39:38 +0000 Subject: [PATCH 318/391] test(fuzz): add a libFuzzer target for the WebSocket UTF-8 check nxt_h1p_ws_utf8_validate() checks the text of WebSocket text messages and close reasons. The client controls all of its input: the text, the masking key and how the message is split into frames. The check unmasks bytes as it reads them, walks a frame that spans several buffers, and keeps a character that a frame border cut for the next frame. No fuzz target reached it. fuzz_http_ws_utf8 splits the payload into frames of 0 to 63 bytes, masks each frame into a chain of up to 8 buffers, and puts a few header bytes before the payload in the first buffer. Byte 0 selects a close reason (a code of 2 bytes, then text, in one frame), byte 1 seeds the cuts, bytes 2-5 are the masking key. The target compares the verdict after each frame with a small validator written from Unicode Table 3-7. RFC 6455 Section 8.1 says that the connection must fail as soon as the text cannot become valid. Thus the verdict must be the same for each frame, not only at the end of the message. A difference causes an abort. The harness includes nxt_h1proto_websocket.c to get to the static function, in the same way as the h1p targets include nxt_h1proto.c. The seed files have names that tell what they hold. The hand-written seeds have plain names. The seeds that a libFuzzer merge kept are min____NN.bin, for example min_text_bad_cont_ascii-2b_01.bin. This target does not reach the close code check, the RSV bits or the opcode rules. They are in the frame header handler, which needs a connection and a request. Co-Authored-By: Claude Opus 5.5 --- auto/fuzzing | 10 + auto/sources | 1 + .../fuzz_http_ws_utf8_seed_corpus/ascii.bin | Bin 0 -> 23 bytes .../beyond_max.bin | Bin 0 -> 10 bytes .../close_bad_reason.bin | 2 + .../close_code_only.bin | 1 + .../close_reason.bin | 1 + .../four_byte.bin | Bin 0 -> 14 bytes .../fuzz_http_ws_utf8_seed_corpus/kosme.bin | Bin 0 -> 17 bytes .../kosme_surrogate.bin | Bin 0 -> 26 bytes .../long_mixed.bin | Bin 0 -> 222 bytes .../min_close_bad_cont_ascii-3b_01.bin | Bin 0 -> 28 bytes .../min_close_bad_start_ascii-3b_01.bin | 1 + .../min_close_valid_empty_01.bin | 2 + .../min_close_valid_empty_02.bin | 1 + .../min_text_bad_cont_ascii-2b-3b-4b_01.bin | Bin 0 -> 130 bytes .../min_text_bad_cont_ascii-2b-3b-4b_02.bin | Bin 0 -> 49 bytes .../min_text_bad_cont_ascii-2b-3b-4b_03.bin | Bin 0 -> 95 bytes .../min_text_bad_cont_ascii-2b-3b-4b_04.bin | Bin 0 -> 76 bytes .../min_text_bad_cont_ascii-2b-3b-4b_05.bin | Bin 0 -> 94 bytes .../min_text_bad_cont_ascii-2b-3b_01.bin | Bin 0 -> 20 bytes .../min_text_bad_cont_ascii-2b-4b_01.bin | Bin 0 -> 12 bytes .../min_text_bad_cont_ascii-2b-4b_02.bin | Bin 0 -> 20 bytes .../min_text_bad_cont_ascii-2b-4b_03.bin | Bin 0 -> 46 bytes .../min_text_bad_cont_ascii-2b-4b_04.bin | Bin 0 -> 21 bytes .../min_text_bad_cont_ascii-2b_01.bin | Bin 0 -> 59 bytes .../min_text_bad_cont_ascii-3b_01.bin | Bin 0 -> 10 bytes .../min_text_bad_start_2b-3b_01.bin | Bin 0 -> 15 bytes .../min_text_bad_start_2b_01.bin | Bin 0 -> 9 bytes .../min_text_bad_start_ascii-2b-3b-4b_01.bin | Bin 0 -> 81 bytes .../min_text_bad_start_ascii-2b-3b-4b_02.bin | Bin 0 -> 96 bytes .../min_text_bad_start_ascii-2b-3b-4b_03.bin | Bin 0 -> 69 bytes .../min_text_bad_start_ascii-2b_01.bin | Bin 0 -> 25 bytes .../min_text_bad_start_ascii-2b_02.bin | Bin 0 -> 45 bytes .../min_text_bad_start_ascii-2b_03.bin | Bin 0 -> 17 bytes .../min_text_bad_start_ascii_01.bin | Bin 0 -> 138 bytes .../min_text_bad_start_ascii_02.bin | Bin 0 -> 130 bytes .../min_text_bad_start_ascii_03.bin | Bin 0 -> 11 bytes .../min_text_bad_start_ascii_04.bin | Bin 0 -> 19 bytes .../min_text_truncated_4b_01.bin | Bin 0 -> 7 bytes .../min_text_valid_ascii-2b-3b-4b_01.bin | Bin 0 -> 201 bytes .../min_text_valid_ascii-2b_01.bin | Bin 0 -> 26 bytes .../min_text_valid_ascii-2b_02.bin | Bin 0 -> 154 bytes .../min_text_valid_ascii-2b_03.bin | Bin 0 -> 202 bytes .../min_text_valid_ascii_01.bin | Bin 0 -> 13 bytes .../min_text_valid_ascii_02.bin | Bin 0 -> 7 bytes .../min_text_valid_empty_01.bin | Bin 0 -> 6 bytes .../overlong.bin | Bin 0 -> 11 bytes .../truncated.bin | Bin 0 -> 7 bytes .../zero_mask.bin | Bin 0 -> 94 bytes fuzzing/nxt_http_ws_utf8_fuzz.c | 324 ++++++++++++++++++ 51 files changed, 343 insertions(+) create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/ascii.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/beyond_max.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/close_bad_reason.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/close_code_only.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/close_reason.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/four_byte.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/kosme.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/kosme_surrogate.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/long_mixed.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_cont_ascii-3b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_start_ascii-3b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_02.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_02.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_03.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_04.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_05.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_02.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_03.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_04.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-3b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_2b-3b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_2b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_02.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b-3b-4b_03.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_02.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_03.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_02.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_03.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_04.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_truncated_4b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b-3b-4b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_02.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_03.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_02.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_empty_01.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/overlong.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/truncated.bin create mode 100644 fuzzing/fuzz_http_ws_utf8_seed_corpus/zero_mask.bin create mode 100644 fuzzing/nxt_http_ws_utf8_fuzz.c diff --git a/auto/fuzzing b/auto/fuzzing index 2d8499d40..6f68f6c14 100644 --- a/auto/fuzzing +++ b/auto/fuzzing @@ -25,6 +25,7 @@ fuzz: $NXT_BUILD_DIR/fuzz_basic \\ $NXT_BUILD_DIR/fuzz_http_h1p \\ $NXT_BUILD_DIR/fuzz_http_h1p_peer \\ $NXT_BUILD_DIR/fuzz_http_h1p_peer_response \\ + $NXT_BUILD_DIR/fuzz_http_ws_utf8 \\ $NXT_BUILD_DIR/fuzz_json \\ $NXT_BUILD_DIR/fuzz_router_app_response \\ $NXT_BUILD_DIR/fuzz_unit_msg @@ -83,6 +84,15 @@ $NXT_BUILD_DIR/fuzz_http_h1p_peer_response: \$(NXT_FUZZ_OBJS) \\ $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ $NXT_FUZZ +$NXT_BUILD_DIR/fuzz_http_ws_utf8: \$(NXT_FUZZ_OBJS) \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC + \$(PP_LD) \$@ + \$(v)\$(NXT_EXEC_LINK) -o $NXT_BUILD_DIR/fuzz_http_ws_utf8 \\ + \$(CFLAGS) $NXT_BUILD_DIR/fuzzing/nxt_http_ws_utf8_fuzz.o \\ + $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\ + $NXT_LD_OPT $NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \\ + $NXT_FUZZ + $NXT_BUILD_DIR/fuzz_json: \$(NXT_FUZZ_OBJS) \\ $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \$(PP_LD) \$@ diff --git a/auto/sources b/auto/sources index a10bab80c..7cb6430ad 100644 --- a/auto/sources +++ b/auto/sources @@ -364,6 +364,7 @@ NXT_FUZZ_SRCS=" \ fuzzing/nxt_http_h1p_fuzz.c \ fuzzing/nxt_http_h1p_peer_fuzz.c \ fuzzing/nxt_http_h1p_peer_response_fuzz.c \ + fuzzing/nxt_http_ws_utf8_fuzz.c \ fuzzing/nxt_json_fuzz.c \ fuzzing/nxt_router_app_response_fuzz.c \ fuzzing/nxt_unit_msg_fuzz.c \ diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/ascii.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/ascii.bin new file mode 100644 index 0000000000000000000000000000000000000000..af11e225ca7fb177d725ba99dbdebd185c0c28ee GIT binary patch literal 23 ecmZQzH2yet1ldq!?o|+V#pPZdqq6h#~5eIny literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/beyond_max.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/beyond_max.bin new file mode 100644 index 0000000000000000000000000000000000000000..41167b29b4da0bb3903255eae85d1b8ce5aebd20 GIT binary patch literal 10 RcmZQzH~*z*`(;8y0{{`u1RDSV literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_bad_reason.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_bad_reason.bin new file mode 100644 index 000000000..a84765e3d --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_bad_reason.bin @@ -0,0 +1,2 @@ + +7ú!=èÿ \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_code_only.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_code_only.bin new file mode 100644 index 000000000..06180a038 --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_code_only.bin @@ -0,0 +1 @@ + 7ú!=è \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_reason.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_reason.bin new file mode 100644 index 000000000..a386754cf --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/close_reason.bin @@ -0,0 +1 @@ + 7ú!=èκόσμε \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/four_byte.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/four_byte.bin new file mode 100644 index 0000000000000000000000000000000000000000..5aeb69dea7caac00a408c85315c735125d2739e1 GIT binary patch literal 14 VcmZQzHUFh(`(Z*u!)QnlsVSKysVM-tfeuXo literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/long_mixed.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/long_mixed.bin new file mode 100644 index 0000000000000000000000000000000000000000..ea43304a15898479c99507f5a31b0775b559b8f6 GIT binary patch literal 222 scmZSJG5@7#n^>HjnW^w-Mzg} literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_start_ascii-3b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_start_ascii-3b_01.bin new file mode 100644 index 000000000..3ad279b27 --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_bad_start_ascii-3b_01.bin @@ -0,0 +1 @@ +Š7\77777777777777777777777777ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ77!!èÿ \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_01.bin new file mode 100644 index 000000000..907417b27 --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_01.bin @@ -0,0 +1,2 @@ + +7ú!èÿ \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_02.bin new file mode 100644 index 000000000..6a65d311b --- /dev/null +++ b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_close_valid_empty_02.bin @@ -0,0 +1 @@ + < \ No newline at end of file diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_01.bin new file mode 100644 index 0000000000000000000000000000000000000000..0ec1623f6106cee2e4d95c582305d665c18efc4e GIT binary patch literal 130 zcmZSJG5@7#n^>HjnW^w-Mzg}B6BC WCJi@~!T6UV0|N&G1H(K8hC~3789tT( literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_02.bin new file mode 100644 index 0000000000000000000000000000000000000000..34dadffd87f47ad479291ae750be6787ad5e0156 GIT binary patch literal 49 vcmZSJU;u)Q!z;^k4(I12Y%9u1QFt_Gatb3mNSq-jU!nXkOipysaoFMd8t$$qEV|=Fe=(I1Ca+ M(t^a))YfhQ01m|$6951J literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_04.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_04.bin new file mode 100644 index 0000000000000000000000000000000000000000..f3e4787ed47177434b7f3a6291e0a81342b554c4 GIT binary patch literal 76 zcmZQD|E0*t#87^CTTxDm!lOBp6+X}b0 O2mloV#TghFJ^%nVrXxrI literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_05.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b-4b_05.bin new file mode 100644 index 0000000000000000000000000000000000000000..aac3d1a3889fcaa5d91c9bde37f619459d0c5157 GIT binary patch literal 94 ucmZSJU;u)Q!z*)g@)ODrZ!5}4QFt_GvciY?Gn*iy3Is)=ih(Aes|5h+?k!*d literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-3b_01.bin new file mode 100644 index 0000000000000000000000000000000000000000..4be93948889346cfe31b10472c6eb48a198c7690 GIT binary patch literal 20 XcmZQzWPpHkd(LfryPzSpBsB#9E*J*B literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_01.bin new file mode 100644 index 0000000000000000000000000000000000000000..9cef8b702e48de6c1acb016dacc6d8368bb65eac GIT binary patch literal 12 TcmZQzU^V}xX!~LRe%o^Z7lQ>3 literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_02.bin new file mode 100644 index 0000000000000000000000000000000000000000..c11d6c7d83e851d93cc9caccbb5f1b4042f983ab GIT binary patch literal 20 XcmZQzU^V}xXv@d|0oot-+nxgeAtMBP literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_03.bin new file mode 100644 index 0000000000000000000000000000000000000000..7f79da1c8cd9e535ed6b94e6f7efb5bab6942e6b GIT binary patch literal 46 qcmZQzWMBXy5U^)pEJ;1L=iJuMAl^O@0S4?044)S?q-2()rT_rAxC#CM literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_04.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_cont_ascii-2b-4b_04.bin new file mode 100644 index 0000000000000000000000000000000000000000..68e08418f8a40dd532161e47621206b263eba3d7 GIT binary patch literal 21 dcmZQzU^V}xX!~J8Lj#NXmtTtg`}fV!9zwRHh4*>GG;L_&+f65#_ literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii-2b_03.bin new file mode 100644 index 0000000000000000000000000000000000000000..9fa87c43b4e1aae30ec5b3eb82518ca38716d5ba GIT binary patch literal 17 VcmZQzWB>yhr=91U&+R$46#y4v1$Y1e literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_01.bin new file mode 100644 index 0000000000000000000000000000000000000000..1169d79c3394a4de526001f540dc35c4a31a32ef GIT binary patch literal 138 dcmZQzH~%%EK^r?@fCB8YU{Py4awJJL003<;7lHr) literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_02.bin new file mode 100644 index 0000000000000000000000000000000000000000..46ef263faa3ea3b3a5967605f4ac21ef825da3ae GIT binary patch literal 130 WcmZQzH~%%EL7QY?O|lLg8X5pR02hJ) literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_bad_start_ascii_03.bin new file mode 100644 index 0000000000000000000000000000000000000000..d09280971ca271264894fe9e0e29c79139aa7040 GIT binary patch literal 11 ScmY#MV5lV0h2K@C5(}V*+>p literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b-3b-4b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b-3b-4b_01.bin new file mode 100644 index 0000000000000000000000000000000000000000..3f838f649b648aaefd35e7c6e4e7bcc0f0bc7fcf GIT binary patch literal 201 ucmZSJU;u)Q!z*)g@)ODrZ!5}4QFt_GvciY?Gn*iy3Is(J$_E&L918%8QC=Yc literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_01.bin new file mode 100644 index 0000000000000000000000000000000000000000..af3f7af5e5edd8c015595fd2047b0cff5f17fe17 GIT binary patch literal 26 WcmZQzF=u3A0D_FeD|2$vkQe|&od#n7 literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_02.bin new file mode 100644 index 0000000000000000000000000000000000000000..dacd920469646334ffdaf9ef760124d20a8ba9e2 GIT binary patch literal 154 XcmZQzHvgq)dv2HTU;-(bC8;R@@bV9w literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_03.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii-2b_03.bin new file mode 100644 index 0000000000000000000000000000000000000000..fefba96748d6c9ca007721725d2ed4379fe22cb7 GIT binary patch literal 202 fcmZQzHvgq)dv2F7Ie>vgK#m>6=t;>eNlgI&Lhuiq literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_01.bin new file mode 100644 index 0000000000000000000000000000000000000000..f7a8de0e13f80fe6f1ba4fb9ec9a808abfb569a9 GIT binary patch literal 13 RcmZQzWB`HGl+2RU6aWaB0&xHU literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_02.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_ascii_02.bin new file mode 100644 index 0000000000000000000000000000000000000000..01f8ad905d8aca5fca6583c7973ae851f0f3b943 GIT binary patch literal 7 OcmZQjwEZ%np$-5D`~u+s literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_empty_01.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/min_text_valid_empty_01.bin new file mode 100644 index 0000000000000000000000000000000000000000..3cd6113aa42a30845445622f526cfd4e78d32d2e GIT binary patch literal 6 NcmZQz`K4%k4gdwM0xJLj literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/overlong.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/overlong.bin new file mode 100644 index 0000000000000000000000000000000000000000..36705d50f088754f2da542ced2198fc7e81d70f3 GIT binary patch literal 11 ScmZQzGykP%dtm*8hV=j!CIu7# literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/truncated.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/truncated.bin new file mode 100644 index 0000000000000000000000000000000000000000..5144de22e932980cacf260e0d5e25721f8fe9c71 GIT binary patch literal 7 OcmZQzG5@7#dkz2w#R6gg literal 0 HcmV?d00001 diff --git a/fuzzing/fuzz_http_ws_utf8_seed_corpus/zero_mask.bin b/fuzzing/fuzz_http_ws_utf8_seed_corpus/zero_mask.bin new file mode 100644 index 0000000000000000000000000000000000000000..48ae7039093473ffcd4605e9dab6f5e42af211f1 GIT binary patch literal 94 lcmZSJU;u)Q!z*)g@)gPtZ!5}4QFt_GvciY?Gn*(E1pwqXEvWzi literal 0 HcmV?d00001 diff --git a/fuzzing/nxt_http_ws_utf8_fuzz.c b/fuzzing/nxt_http_ws_utf8_fuzz.c new file mode 100644 index 000000000..76020faf5 --- /dev/null +++ b/fuzzing/nxt_http_ws_utf8_fuzz.c @@ -0,0 +1,324 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +#include + +/* DO NOT TRY THIS AT HOME! */ +#include "nxt_h1proto_websocket.c" + + +/* + * This target fuzzes the UTF-8 check that the router does on WebSocket text + * messages and close reasons, nxt_h1p_ws_utf8_validate(). The client + * controls each byte of the input: the text, the mask, and how the message + * is split into frames. The check reads masked bytes, walks a frame that + * spans several buffers, and keeps a character that a frame border cut for + * the next frame. + * + * The target compares the verdict with a small independent validator + * written from the table of well-formed byte sequences in the Unicode + * standard (Table 3-7). RFC 6455 Section 8.1 says that the connection must + * fail as soon as the text cannot become valid. Thus the verdict must be + * the same after each frame, not only at the end of the message. A + * difference causes an abort. + * + * Input layout: + * byte 0 flags; bit 0 checks a close frame reason, not a message + * byte 1 seeds the frame and buffer cuts + * bytes 2-5 the masking key + * the rest the unmasked payload + */ + + +#define KMININPUTLENGTH 6 +#define KMAXINPUTLENGTH 4096 + +#define NXT_WS_FUZZ_MAX_BUFS 8 + + +typedef struct { + uint8_t need; + u_char lo; + u_char hi; + uint8_t failed; +} nxt_ws_fuzz_ref_t; + + +extern int LLVMFuzzerInitialize(int *argc, char ***argv); +extern int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +static nxt_int_t nxt_ws_fuzz_frame(nxt_mp_t *mp, nxt_h1p_ws_utf8_t *state, + const u_char *payload, size_t size, size_t skip, const u_char *mask, + nxt_uint_t final, uint32_t *seed, nxt_int_t *res); +static void nxt_ws_fuzz_ref(nxt_ws_fuzz_ref_t *ref, const u_char *p, + size_t size); +static uint32_t nxt_ws_fuzz_rand(uint32_t *state); + + +extern char **environ; + + +int +LLVMFuzzerInitialize(int *argc, char ***argv) +{ + if (nxt_lib_start("fuzzing", NULL, &environ) != NXT_OK) { + return NXT_ERROR; + } + + return 0; +} + + +int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + size_t len, cut, skip; + uint32_t seed; + nxt_mp_t *mp; + nxt_int_t res, expect; + nxt_uint_t close, final; + const u_char *mask, *p, *end; + nxt_ws_fuzz_ref_t ref; + nxt_h1p_ws_utf8_t state; + + if (size < KMININPUTLENGTH || size > KMAXINPUTLENGTH) { + return 0; + } + + close = data[0] & 1; + seed = (uint32_t) data[1] + 1; + mask = &data[2]; + + p = data + 6; + end = data + size; + + /* A close frame has a code of two bytes before the reason. */ + if (close && end - p < 2) { + return 0; + } + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return 0; + } + + nxt_memzero(&state, sizeof(nxt_h1p_ws_utf8_t)); + nxt_memzero(&ref, sizeof(nxt_ws_fuzz_ref_t)); + + do { + len = end - p; + skip = 0; + + if (close) { + /* A close frame is one complete frame. */ + skip = 2; + + } else { + /* Empty frames are permitted, so a cut can be 0. */ + cut = nxt_ws_fuzz_rand(&seed) % 64; + len = nxt_min(len, cut); + } + + final = (p + len == end); + + if (nxt_ws_fuzz_frame(mp, &state, p, len, skip, mask, final, &seed, + &res) + != NXT_OK) + { + goto done; + } + + nxt_ws_fuzz_ref(&ref, p + skip, len - skip); + + expect = (ref.failed || (final && ref.need != 0)) ? NXT_ERROR : NXT_OK; + + if (res != expect) { + nxt_thread_log_alert("ws utf8 fuzz: frame at %uz of %uz bytes: " + "%i, expected %i", + (size_t) (p - data - 6), len, res, expect); + abort(); + } + + p += len; + + } while (res == NXT_OK && p < end); + +done: + + nxt_mp_destroy(mp); + + return 0; +} + + +/* + * Masks the payload of one frame into a chain of buffers, then runs the + * check on it. Each buffer is a heap allocation of its exact size, so that + * ASan catches a read after the end of any buffer. The first buffer starts + * with a few bytes that stand for the frame header: the check must start at + * "start", not at the buffer. + */ + +static nxt_int_t +nxt_ws_fuzz_frame(nxt_mp_t *mp, nxt_h1p_ws_utf8_t *state, const u_char *payload, + size_t size, size_t skip, const u_char *mask, nxt_uint_t final, + uint32_t *seed, nxt_int_t *res) +{ + size_t hsize, len, cut, i, offset; + u_char *start; + nxt_int_t ret; + nxt_buf_t *b, *first, **prev; + nxt_uint_t n, k; + u_char *mem[NXT_WS_FUZZ_MAX_BUFS]; + + ret = NXT_ERROR; + + first = NULL; + prev = &first; + start = NULL; + + n = 0; + offset = 0; + hsize = nxt_ws_fuzz_rand(seed) % 13 + 2; + + do { + len = size - offset; + + if (n < NXT_WS_FUZZ_MAX_BUFS - 1) { + cut = nxt_ws_fuzz_rand(seed) % 24 + 1; + len = nxt_min(len, cut); + } + + if (n == 0) { + len += hsize; + } + + b = nxt_buf_mem_alloc(mp, 0, 0); + if (b == NULL) { + goto done; + } + + mem[n] = nxt_malloc(len); + if (mem[n] == NULL) { + goto done; + } + + b->mem.start = mem[n++]; + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start + len; + b->mem.end = b->mem.free; + + i = 0; + + if (n == 1) { + nxt_memset(b->mem.start, 0x81, hsize); + start = b->mem.start + hsize; + i = hsize; + } + + for ( /* void */ ; i < len; i++) { + b->mem.start[i] = payload[offset] ^ mask[offset % 4]; + offset++; + } + + *prev = b; + prev = &b->next; + + } while (offset < size); + + *res = nxt_h1p_ws_utf8_validate(state, first, start, skip, size - skip, + mask, final); + + ret = NXT_OK; + +done: + + for (k = 0; k < n; k++) { + nxt_free(mem[k]); + } + + return ret; +} + + +/* Unicode Table 3-7, Well-Formed UTF-8 Byte Sequences. */ + +static void +nxt_ws_fuzz_ref(nxt_ws_fuzz_ref_t *ref, const u_char *p, size_t size) +{ + u_char c; + const u_char *end; + + for (end = p + size; p < end && !ref->failed; p++) { + c = *p; + + if (ref->need != 0) { + if (c < ref->lo || c > ref->hi) { + ref->failed = 1; + return; + } + + ref->need--; + ref->lo = 0x80; + ref->hi = 0xBF; + continue; + } + + ref->lo = 0x80; + ref->hi = 0xBF; + + if (c < 0x80) { + continue; + } + + if (c >= 0xC2 && c <= 0xDF) { + ref->need = 1; + + } else if (c == 0xE0) { + ref->need = 2; + ref->lo = 0xA0; + + } else if (c == 0xED) { + ref->need = 2; + ref->hi = 0x9F; + + } else if (c >= 0xE1 && c <= 0xEF) { + ref->need = 2; + + } else if (c == 0xF0) { + ref->need = 3; + ref->lo = 0x90; + + } else if (c >= 0xF1 && c <= 0xF3) { + ref->need = 3; + + } else if (c == 0xF4) { + ref->need = 3; + ref->hi = 0x8F; + + } else { + ref->failed = 1; + return; + } + } +} + + +/* xorshift32: fast, and the same sequence for the same input. */ + +static uint32_t +nxt_ws_fuzz_rand(uint32_t *state) +{ + uint32_t x; + + x = *state; + + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + + *state = x; + + return x; +} From 4177aa98f203f80654480585b6ef21c4c5bc8b16 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 16:40:18 +0000 Subject: [PATCH 319/391] ci(fuzz): run the three new targets in CI run-ci.sh runs fuzz_http_h1p_peer_response, fuzz_http_chunk and fuzz_http_ws_utf8 by default, each against its own seed corpus. The peer response target gets fuzz_http.dict. The other two get no dictionary: request words do not help a chunked body or UTF-8 text. Ten targets at 60 seconds are about ten minutes, and the wall cap permits two minutes for each target. Thus the pull request timeout goes from 25 to 30 minutes. The path filters already cover the new targets: src/nxt_http* holds the chunk decoder and src/nxt_h1proto* the WebSocket code. README.md, build-fuzz.sh and oss-fuzz.sh list the new targets, and README.md says what each differential target compares. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/fuzzing.yml | 18 +++++++++--------- fuzzing/README.md | 33 +++++++++++++++++++++++++++++++-- fuzzing/build-fuzz.sh | 3 +++ fuzzing/oss-fuzz.sh | 4 ++++ fuzzing/run-ci.sh | 15 ++++++++++++--- 5 files changed, 59 insertions(+), 14 deletions(-) diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index b875a97c2..a3b0994af 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -31,7 +31,7 @@ on: required: false default: '60' targets: - description: 'Space-separated fuzzer names (empty = all seven)' + description: 'Space-separated fuzzer names (empty = all ten)' required: false default: '' @@ -50,13 +50,13 @@ jobs: libfuzzer: name: libFuzzer (ASan + UBSan) runs-on: ubuntu-latest - # Seven targets at a 60s budget is a little over seven minutes -- each + # Ten targets at a 60s budget is a little over ten minutes -- each # finishes within a second or two of its budget, and run-ci.sh caps the - # worst case at two minutes each. The clang/ASan build is a few minutes - # on top. A dispatch can ask for a much longer budget across all seven - # targets, so it gets a much longer bound; a pull request must not, or a - # hung job holds a runner for an hour. - timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 120 || 25 }} + # worst case at two minutes each, twenty in all. The clang/ASan build is + # a few minutes on top. A dispatch can ask for a much longer budget + # across all ten targets, so it gets a much longer bound; a pull request + # must not, or a hung job holds a runner for an hour. + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 120 || 30 }} steps: - uses: actions/checkout@v5 @@ -87,11 +87,11 @@ jobs: ./configure --no-regex --no-pcre2 --fuzz=-fsanitize=fuzzer make fuzz -j"$(nproc)" - # All seven targets, on a pull request and on a dispatch alike. Picking a + # All ten targets, on a pull request and on a dispatch alike. Picking a # subset per changed path was tried and dropped: every path filter above # already maps to a target, `fuzzing/**` maps to all of them, and a # target left out of the list is a harness that CI compiles but never - # runs. At about 61 seconds each the whole set is close to seven minutes. + # runs. At about 61 seconds each the whole set is close to ten minutes. - name: Run the fuzzers env: FUZZ_SECONDS: ${{ github.event.inputs.seconds || '60' }} diff --git a/fuzzing/README.md b/fuzzing/README.md index 1a2813d68..2ce2a02ba 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -38,22 +38,51 @@ $ make fuzz -j$(nproc) ```shell $ mkdir -p build/fuzz_basic_seed +$ mkdir -p build/fuzz_http_chunk_seed $ mkdir -p build/fuzz_http_controller_seed $ mkdir -p build/fuzz_http_h1p_seed $ mkdir -p build/fuzz_http_h1p_peer_seed +$ mkdir -p build/fuzz_http_h1p_peer_response_seed +$ mkdir -p build/fuzz_http_ws_utf8_seed $ mkdir -p build/fuzz_json_seed $ mkdir -p build/fuzz_router_app_response_seed $ mkdir -p build/fuzz_unit_msg_seed $ ./build/fuzz_basic build/fuzz_basic_seed fuzzing/fuzz_basic_seed_corpus +$ ./build/fuzz_http_chunk build/fuzz_http_chunk_seed fuzzing/fuzz_http_chunk_seed_corpus $ ./build/fuzz_http_controller build/fuzz_http_controller_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_http_h1p build/fuzz_http_h1p_seed fuzzing/fuzz_http_seed_corpus $ ./build/fuzz_http_h1p_peer build/fuzz_http_h1p_peer_seed fuzzing/fuzz_http_seed_corpus +$ ./build/fuzz_http_h1p_peer_response \ + build/fuzz_http_h1p_peer_response_seed \ + fuzzing/fuzz_http_h1p_peer_response_seed_corpus +$ ./build/fuzz_http_ws_utf8 build/fuzz_http_ws_utf8_seed fuzzing/fuzz_http_ws_utf8_seed_corpus $ ./build/fuzz_json build/fuzz_json_seed fuzzing/fuzz_json_seed_corpus $ ./build/fuzz_router_app_response build/fuzz_router_app_response_seed fuzzing/fuzz_router_app_response_seed_corpus $ ./build/fuzz_unit_msg build/fuzz_unit_msg_seed fuzzing/fuzz_unit_msg_seed_corpus ``` +### Differential targets. + +Three targets check a result, not only that the code does not crash. Each +one aborts on a difference, so a wrong answer is reported like a crash. + +- `fuzz_http_chunk` decodes a chunked body two times: one time as a single + buffer, and one time cut into pieces given to the decoder in chains of one + to three buffers. The two runs must give the same verdict, the same end + of the message, and the same decoded bytes. The first input byte seeds + the cuts. +- `fuzz_http_h1p_peer_response` parses an upstream response header in the + same way as the proxy: it continues after each read that returned + `NXT_AGAIN`. A run that gets the header a few bytes at a time must give + the same status, header size and fields as a run that gets it in one pass. + The first input byte seeds the read sizes. +- `fuzz_http_ws_utf8` runs the UTF-8 check for WebSocket text messages and + close reasons on masked frames that are split into several buffers. It + compares the verdict after each frame with a small validator written from + Unicode Table 3-7. Byte 0 selects a close reason, byte 1 seeds the cuts, + bytes 2-5 are the masking key. + Here is more information about [LibFuzzer](https://llvm.org/docs/LibFuzzer.html). ## Build fuzzers using other fuzzing engines. @@ -90,13 +119,13 @@ UBSan and runs `fuzzing/run-ci.sh`, which gives each target a bounded libFuzzer budget against its seed corpus and fails on the first crash. ```shell -$ fuzzing/run-ci.sh # all seven, 60s each +$ fuzzing/run-ci.sh # all ten, 60s each $ fuzzing/run-ci.sh -t 120 fuzz_http_h1p # one target, longer ``` A pull request that touches `src/nxt_http*`, `src/nxt_h1proto*`, `src/nxt_controller*`, `src/nxt_conf*`, `src/nxt_unit*`, `src/nxt_router*` or -`fuzzing/` runs all seven targets -- roughly seven minutes; a manual +`fuzzing/` runs all ten targets -- roughly ten minutes; a manual `workflow_dispatch` takes a budget and a target list as inputs. Reproducers land in `build/fuzz-artifacts/` and are uploaded as an artifact when the job fails. diff --git a/fuzzing/build-fuzz.sh b/fuzzing/build-fuzz.sh index 7f11e256b..72820cb0c 100644 --- a/fuzzing/build-fuzz.sh +++ b/fuzzing/build-fuzz.sh @@ -10,9 +10,12 @@ export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" make fuzz -j$(nproc) mkdir -p build/fuzz_basic_seed +mkdir -p build/fuzz_http_chunk_seed mkdir -p build/fuzz_http_controller_seed mkdir -p build/fuzz_http_h1p_seed mkdir -p build/fuzz_http_h1p_peer_seed +mkdir -p build/fuzz_http_h1p_peer_response_seed +mkdir -p build/fuzz_http_ws_utf8_seed mkdir -p build/fuzz_json_seed mkdir -p build/fuzz_router_app_response_seed mkdir -p build/fuzz_unit_msg_seed diff --git a/fuzzing/oss-fuzz.sh b/fuzzing/oss-fuzz.sh index 1fe5ecda8..abd142f8a 100644 --- a/fuzzing/oss-fuzz.sh +++ b/fuzzing/oss-fuzz.sh @@ -12,6 +12,7 @@ pushd fuzzing/ cp fuzz_http.dict $OUT/fuzz_http_controller.dict cp fuzz_http.dict $OUT/fuzz_http_h1p.dict cp fuzz_http.dict $OUT/fuzz_http_h1p_peer.dict +cp fuzz_http.dict $OUT/fuzz_http_h1p_peer_response.dict # Create temporary directories. cp -r fuzz_http_seed_corpus/ fuzz_http_controller_seed_corpus/ @@ -23,6 +24,9 @@ zip -r $OUT/fuzz_http_controller_seed_corpus.zip fuzz_http_controller_seed_corp zip -r $OUT/fuzz_http_h1p_seed_corpus.zip fuzz_http_h1p_seed_corpus/ zip -r $OUT/fuzz_http_h1p_peer_seed_corpus.zip fuzz_http_h1p_peer_seed_corpus/ zip -r $OUT/fuzz_json_seed_corpus.zip fuzz_json_seed_corpus/ +zip -r $OUT/fuzz_http_chunk_seed_corpus.zip fuzz_http_chunk_seed_corpus/ +zip -r $OUT/fuzz_http_h1p_peer_response_seed_corpus.zip fuzz_http_h1p_peer_response_seed_corpus/ +zip -r $OUT/fuzz_http_ws_utf8_seed_corpus.zip fuzz_http_ws_utf8_seed_corpus/ # Delete temporary directories. rm -r fuzz_http_controller_seed_corpus/ fuzz_http_h1p_seed_corpus/ fuzz_http_h1p_peer_seed_corpus/ diff --git a/fuzzing/run-ci.sh b/fuzzing/run-ci.sh index 38b6f6e0f..61e0127ef 100755 --- a/fuzzing/run-ci.sh +++ b/fuzzing/run-ci.sh @@ -10,7 +10,7 @@ # fuzzing/run-ci.sh [-t SECONDS] [TARGET ...] # # -t SECONDS libFuzzer budget per target (default 60) -# TARGET ... fuzzer names; default is all seven +# TARGET ... fuzzer names; default is all ten # # Expects the fuzzers to be built already: # CC=clang CXX=clang++ \ @@ -42,12 +42,15 @@ case "$BUDGET" in ''|*[!0-9]*) die "budget must be a whole number of seconds, got '$BUDGET'" ;; esac -# Seed corpus and dictionary per target. fuzz_http_* share the HTTP corpus. +# Seed corpus and dictionary per target. The fuzz_http_* targets that take a +# request share the HTTP corpus. The other targets have a corpus of their own. corpus_for() { case "$1" in fuzz_basic) echo fuzz_basic_seed_corpus ;; fuzz_json) echo fuzz_json_seed_corpus ;; fuzz_unit_msg) echo fuzz_unit_msg_seed_corpus ;; + fuzz_http_chunk | fuzz_http_h1p_peer_response | fuzz_http_ws_utf8) + echo "$1_seed_corpus" ;; fuzz_router_app_response) echo fuzz_router_app_response_seed_corpus ;; fuzz_http_*) echo fuzz_http_seed_corpus ;; @@ -55,8 +58,12 @@ corpus_for() { esac } +# fuzz_http.dict holds request words. They also help the upstream response +# header, but not a chunked body or WebSocket text. dict_for() { case "$1" in + fuzz_http_chunk | fuzz_http_ws_utf8) + echo "" ;; fuzz_http_*) echo fuzz_http.dict ;; *) echo "" ;; esac @@ -64,7 +71,9 @@ dict_for() { [ $# -gt 0 ] || set -- fuzz_basic fuzz_json fuzz_http_controller \ fuzz_http_h1p fuzz_http_h1p_peer \ - fuzz_router_app_response fuzz_unit_msg + fuzz_http_h1p_peer_response fuzz_http_chunk \ + fuzz_http_ws_utf8 fuzz_router_app_response \ + fuzz_unit_msg # Freeze the list: the loop below rebuilds "$@" to hold each target's libFuzzer # arguments, so it cannot also be the list being iterated. From 434a7489fb0ee3f6c0bea59d962461bc03fe377a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 17:35:59 +0000 Subject: [PATCH 320/391] changelog: unitctl reads json and pem, not json5 The hjson and YAML entry says "Supported input formats are now json, json5 and pem". A later change in the same release removes JSON5 too, and its entry says so: unitctl refuses a ".json5" file by name (to_config_body() in tools/unitctl/unitctl/src/inputfile.rs). So the 1.37.0 notes said both. Correct the older entry. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 2 +- docs/changes.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 4b8b64bb4..a5333de10 100644 --- a/CHANGES +++ b/CHANGES @@ -487,7 +487,7 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 misleading error at the first comment. A ".yaml" or ".yml" file is refused with "YAML is no longer supported: convert the file to JSON first, for example with "yq -o=json"". Supported input formats are - now json, json5 and pem. "unitctl status -t yaml" is gone; + now json and pem. "unitctl status -t yaml" is gone; --output-format now accepts json, json-pretty and text. This removes the nu-json dependency and the 30 crates of Nushell utility code it pulled in for a tool that speaks one JSON control API. It also diff --git a/docs/changes.xml b/docs/changes.xml index edf8b681b..b82ecc855 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -650,8 +650,8 @@ JSON, so piping JSON still works, but comments and unquoted keys do not. A convert the file to JSON first", instead of being handed to the JSON parser, which reports a misleading error at the first comment. A ".yaml" or ".yml" file is refused with "YAML is no longer supported: convert the file to JSON -first, for example with "yq -o=json"". Supported input formats are now json, -json5 and pem. "unitctl status -t yaml" is gone; --output-format now accepts +first, for example with "yq -o=json"". Supported input formats are now json +and pem. "unitctl status -t yaml" is gone; --output-format now accepts json, json-pretty and text. This removes the nu-json dependency and the 30 crates of Nushell utility code it pulled in for a tool that speaks one JSON control API. It also removes serde_yaml and unsafe-libyaml; the latter is a From 3e7ea2140c1298cf8b4516098d48a9cd0464ae64 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 12:43:36 +0000 Subject: [PATCH 321/391] router: accept control messages only from main and the controller An application process holds the write end of the router main port. The port dispatches on the type byte that the sender chooses. Thus an application could send control messages to the router. It could make the router quit, load a configuration, restart applications, remove pids, redirect the error log, and reopen the access log. The router main port has no shared memory queue. Thus each message on it comes through the socket, and the kernel gives the sender pid. The router now compares this pid with the pid of the correct sender: - QUIT, CHANGE_FILE and ACCESS_LOG: main. - REMOVE_PID: main, or a registered prototype. The router cannot bind the pid to the children of that prototype, because a child that died during its start has no record in the router. - DATA, APP_RESTART and STATUS: the controller. The router reads the ports of main and the controller at message time, because a restarted router learns them again through NEW_PORT. For a refused message, the router writes one alert and closes the descriptors. It does not send a reply, because the sender chooses the reply port and the stream. This change does not gate RPC replies, NEW_PORT, MMAP, GET_MMAP, GET_PORT and OOSM. Later changes will do this. Thus the gate makes the attack harder, but does not stop it yet. The check compares the sender pid with the pid of rt->port_by_type[type]. NEW_PORT sets that slot for the type in the message, from the pid that the message gives. An application can first send a NEW_PORT with type MAIN and its own pid, and then QUIT. The NEW_PORT step of #341 (see also #223) closes this. The CHANGES entry says so. On platforms without sender credentials, the pid falls back to the pid in the message header. The sender chooses that pid, so the check is weaker there. Main has the same limit. The new unit test sends each gated type through the router port table from each false sender, with and without the ports of main and the controller. It checks that the router refuses the message, closes its descriptors, and does not run the handler. It also checks that the correct senders pass. The remove pid soak test now sends its REMOVE_PID as main. Refs #341 Co-Authored-By: Claude Opus 5.5 --- CHANGES | 13 + auto/sources | 1 + docs/changes.xml | 15 + src/nxt_router.c | 204 ++++++++- src/nxt_router.h | 9 + src/test/nxt_router_remove_pid_soak_test.c | 31 +- src/test/nxt_router_sender_test.c | 498 +++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + 9 files changed, 772 insertions(+), 4 deletions(-) create mode 100644 src/test/nxt_router_sender_test.c diff --git a/CHANGES b/CHANGES index a5333de10..19078569d 100644 --- a/CHANGES +++ b/CHANGES @@ -44,6 +44,19 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 spans are children of FreeUnit's span. Before, the application got the client's traceparent unchanged. + *) Security: an application process could send control messages to + the router. It could make the router quit, load a configuration, + restart applications, forget a process, redirect the error log, or + reopen the access log. The router now checks the sender pid that + the kernel gives. QUIT, CHANGE_FILE and ACCESS_LOG must come from + main. REMOVE_PID must come from main or from a prototype. DATA, + APP_RESTART and STATUS must come from the controller. The router + closes the descriptors of a refused message and does not reply. + This makes the attack harder, but does not stop it yet: the router + reads the pid of main and of the controller from ports that NEW_PORT + registers, and an application can still send a false NEW_PORT + first. A later change checks NEW_PORT. + *) Bugfix: when unitd ran as root and the control socket was given to another user or group with --control-user or --control-group, that user passed the socket file permissions but the controller then diff --git a/auto/sources b/auto/sources index 7cb6430ad..36566d6eb 100644 --- a/auto/sources +++ b/auto/sources @@ -198,6 +198,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_app_timeout_test.c \ src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_router_detached_test.c \ + src/test/nxt_router_sender_test.c \ src/test/nxt_router_websocket_test.c \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index b82ecc855..f82ec97b2 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -112,6 +112,21 @@ unchanged. + + +an application process could send control messages to the router. It could +make the router quit, load a configuration, restart applications, forget a +process, redirect the error log, or reopen the access log. The router now +checks the sender pid that the kernel gives. QUIT, CHANGE_FILE and ACCESS_LOG +must come from main. REMOVE_PID must come from main or from a prototype. +DATA, APP_RESTART and STATUS must come from the controller. The router closes +the descriptors of a refused message and does not reply. This makes the +attack harder, but does not stop it yet: the router reads the pid of main and +of the controller from ports that NEW_PORT registers, and an application can +still send a false NEW_PORT first. A later change checks NEW_PORT. + + + an application could write an unexpected value into an entry of a shared diff --git a/src/nxt_router.c b/src/nxt_router.c index b7f4f9e4e..07650a82f 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -214,6 +214,22 @@ static void nxt_router_greet_controller(nxt_task_t *task, static nxt_int_t nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app); +static nxt_bool_t nxt_router_msg_from(nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_type_t type); +static nxt_bool_t nxt_router_msg_sender_is(nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_type_t type); +static nxt_bool_t nxt_router_msg_sender_is_proto(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_msg_sender_refuse(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_main_only(nxt_task_t *task, nxt_port_recv_msg_t *msg, + nxt_port_handler_t handler); +static void nxt_router_quit_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_change_file_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static void nxt_router_access_log_handler(nxt_task_t *task, + nxt_port_recv_msg_t *msg); static void nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_router_app_restart_handler(nxt_task_t *task, @@ -436,17 +452,17 @@ static const nxt_str_t *nxt_app_msg_prefix[] = { static const nxt_port_handlers_t nxt_router_process_port_handlers = { - .quit = nxt_signal_quit_handler, + .quit = nxt_router_quit_handler, .new_port = nxt_router_new_port_handler, .get_port = nxt_router_get_port_handler, - .change_file = nxt_port_change_log_file_handler, + .change_file = nxt_router_change_file_handler, .mmap = nxt_port_mmap_handler, .get_mmap = nxt_router_get_mmap_handler, .data = nxt_router_conf_data_handler, .app_restart = nxt_router_app_restart_handler, .status = nxt_router_status_handler, .remove_pid = nxt_router_remove_pid_handler, - .access_log = nxt_router_access_log_reopen_handler, + .access_log = nxt_router_access_log_handler, .rpc_ready = nxt_port_rpc_handler, .rpc_error = nxt_port_rpc_handler, .oosm = nxt_router_oosm_handler, @@ -1534,6 +1550,166 @@ nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Each application process has the write end of the main port of the + * router, and the sender sets the message type. Thus the router accepts + * these control messages only from their normal sender: + * + * - QUIT, CHANGE_FILE and ACCESS_LOG: main. + * - REMOVE_PID: main or a prototype. + * - DATA, APP_RESTART and STATUS: the controller. + * + * The main port of the router has no shared memory queue. Thus each + * message comes through the socket, and the kernel gives the pid of the + * sender (SCM_CREDENTIALS). Without SCM_CREDENTIALS, the pid comes from + * the message header, which the sender sets. + * + * The router reads the ports from rt->port_by_type[] at message time, + * because a restarted router gets them again through NEW_PORT. A NULL + * port refuses the message. + * + * The check is only as good as rt->port_by_type[]. NEW_PORT sets that + * slot from the message, and the router does not check its sender yet. + * Thus an application can first send a NEW_PORT with type MAIN and its + * own pid. The NEW_PORT step of #341 closes this. + */ + +static nxt_bool_t +nxt_router_msg_from(nxt_task_t *task, nxt_port_recv_msg_t *msg, + nxt_process_type_t type) +{ + nxt_port_t *port; + + port = task->thread->runtime->port_by_type[type]; + + return (port != NULL && nxt_recv_msg_cmsg_pid(msg) == port->pid); +} + + +static nxt_bool_t +nxt_router_msg_sender_is(nxt_task_t *task, nxt_port_recv_msg_t *msg, + nxt_process_type_t type) +{ + if (nxt_fast_path(nxt_router_msg_from(task, msg, type))) { + return 1; + } + + nxt_router_msg_sender_refuse(task, msg); + + return 0; +} + + +/* + * A prototype sends REMOVE_PID for a child that it forked + * (nxt_proto_child_exited()). The router cannot check that the pid is a + * child of that prototype: a child that stopped during its start has no + * record in the router. Thus any registered prototype passes. + */ + +static nxt_bool_t +nxt_router_msg_sender_is_proto(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_bool_t proto; + nxt_process_t *process; + + if (nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN)) { + return 1; + } + + process = nxt_runtime_process_ref(task->thread->runtime, + nxt_recv_msg_cmsg_pid(msg)); + + if (process != NULL) { + proto = (nxt_process_type(process) == NXT_PROCESS_PROTOTYPE); + + nxt_process_use(task, process, -1); + + if (proto) { + return 1; + } + } + + nxt_router_msg_sender_refuse(task, msg); + + return 0; +} + + +/* + * No reply: the sender selects the reply port and the stream, so a reply + * would let a false sender make the router write to any stream. + */ + +static void +nxt_router_msg_sender_refuse(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_alert(task, "process %PI sent message type %uD claiming process %PI; " + "refused", nxt_recv_msg_cmsg_pid(msg), + (uint32_t) msg->port_msg.type, msg->port_msg.pid); + + nxt_port_recv_msg_close_fds(msg); + +#if (NXT_TESTS) + nxt_router_test_senders_refused++; +#endif +} + + +/* Other processes share these handlers, so the router table gets wrappers. */ + +static void +nxt_router_main_only(nxt_task_t *task, nxt_port_recv_msg_t *msg, + nxt_port_handler_t handler) +{ + if (nxt_fast_path(nxt_router_msg_sender_is(task, msg, NXT_PROCESS_MAIN))) { + handler(task, msg); + } +} + + +static void +nxt_router_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_router_main_only(task, msg, nxt_signal_quit_handler); +} + + +static void +nxt_router_change_file_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_router_main_only(task, msg, nxt_port_change_log_file_handler); +} + + +static void +nxt_router_access_log_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_router_main_only(task, msg, nxt_router_access_log_reopen_handler); +} + + +#if (NXT_TESTS) + +/* For src/test/nxt_router_sender_test.c. */ + +nxt_uint_t nxt_router_test_senders_refused; + + +nxt_bool_t +nxt_router_test_msg_sender_is(nxt_task_t *task, nxt_port_recv_msg_t *msg, + nxt_process_type_t type) +{ + if (type == NXT_PROCESS_PROTOTYPE) { + return nxt_router_msg_sender_is_proto(task, msg); + } + + return nxt_router_msg_sender_is(task, msg, type); +} + +#endif + + static void nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -1543,6 +1719,12 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_t *port; nxt_router_temp_conf_t *tmcf; + if (nxt_slow_path(!nxt_router_msg_sender_is(task, msg, + NXT_PROCESS_CONTROLLER))) + { + return; + } + port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, msg->port_msg.reply_port); @@ -1640,6 +1822,12 @@ nxt_router_app_restart_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_t *proto_port; nxt_port_msg_type_t reply; + if (nxt_slow_path(!nxt_router_msg_sender_is(task, msg, + NXT_PROCESS_CONTROLLER))) + { + return; + } + reply_port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, msg->port_msg.reply_port); @@ -1740,6 +1928,12 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_event_engine_t *engine; nxt_status_report_t *report; + if (nxt_slow_path(!nxt_router_msg_sender_is(task, msg, + NXT_PROCESS_CONTROLLER))) + { + return; + } + port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, msg->port_msg.reply_port); @@ -1853,6 +2047,10 @@ nxt_router_remove_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_event_engine_t *engine; + if (nxt_slow_path(!nxt_router_msg_sender_is_proto(task, msg))) { + return; + } + nxt_port_remove_pid_handler(task, msg); nxt_queue_each(engine, &nxt_router->engines, nxt_event_engine_t, link0) diff --git a/src/nxt_router.h b/src/nxt_router.h index c593a8ade..18fd7f2ff 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -363,6 +363,15 @@ void nxt_router_access_log_reopen_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); #if (NXT_TESTS) +/* + * The sender checks of the router main port, for the router sender test. + * NXT_PROCESS_PROTOTYPE selects the REMOVE_PID check: main or a prototype. + */ +nxt_bool_t nxt_router_test_msg_sender_is(nxt_task_t *task, + nxt_port_recv_msg_t *msg, nxt_process_type_t type); + +extern nxt_uint_t nxt_router_test_senders_refused; + /* The detached edge handler, for the sender check test. */ void nxt_router_test_detached_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/src/test/nxt_router_remove_pid_soak_test.c b/src/test/nxt_router_remove_pid_soak_test.c index 72c288037..0f697d954 100644 --- a/src/test/nxt_router_remove_pid_soak_test.c +++ b/src/test/nxt_router_remove_pid_soak_test.c @@ -115,6 +115,11 @@ nxt_router_remove_pid_soak_msg(nxt_port_recv_msg_t *msg, nxt_buf_t *buf, msg->port_msg.pid = nxt_pid; msg->port_msg.type = _NXT_PORT_MSG_REMOVE_PID; msg->port_msg.last = 1; + +#if (NXT_USE_CMSG_PID) + /* From main; see main_port in the test. */ + msg->cmsg_pid = nxt_pid; +#endif } @@ -130,7 +135,7 @@ nxt_router_remove_pid_soak_test(nxt_thread_t *thr) nxt_uint_t i; nxt_bool_t app_mutex; nxt_task_t *task; - nxt_port_t *router_port, *dport; + nxt_port_t *router_port, *dport, *main_port; nxt_router_t router, *saved_router; nxt_runtime_t *rt, *saved_rt; nxt_app_joint_t *joint; @@ -144,6 +149,7 @@ nxt_router_remove_pid_soak_test(nxt_thread_t *thr) ret = NXT_ERROR; router_port = NULL; dport = NULL; + main_port = NULL; joint = NULL; app_mutex = 0; app = &nxt_router_remove_pid_soak_test_app; @@ -244,6 +250,23 @@ nxt_router_remove_pid_soak_test(nxt_thread_t *thr) dport->pair[1] = -1; dport->socket.fd = -1; + /* + * The router accepts REMOVE_PID only from main or a prototype (issue + * #341). Main sends it with a start stream when a prototype dies, so + * the message comes from main: a main port with the pid of this process. + */ + + main_port = nxt_port_new(task, 2, nxt_pid, NXT_PROCESS_MAIN); + if (nxt_slow_path(main_port == NULL)) { + goto done; + } + + main_port->pair[0] = -1; + main_port->pair[1] = -1; + main_port->socket.fd = -1; + + rt->port_by_type[NXT_PROCESS_MAIN] = main_port; + nxt_memzero(app, sizeof(nxt_app_t)); if (nxt_slow_path(nxt_thread_mutex_create(&app->mutex) != NXT_OK)) { @@ -407,6 +430,12 @@ nxt_router_remove_pid_soak_test(nxt_thread_t *thr) nxt_port_use(task, router_port, -1); } + if (main_port != NULL) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + + nxt_port_use(task, main_port, -1); + } + thr->engine = saved_engine; thr->runtime = saved_rt; nxt_router = saved_router; diff --git a/src/test/nxt_router_sender_test.c b/src/test/nxt_router_sender_test.c new file mode 100644 index 000000000..adb447647 --- /dev/null +++ b/src/test/nxt_router_sender_test.c @@ -0,0 +1,498 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Test for the sender checks on the main port of the router + * (src/nxt_router.c, issue #341). The router accepts these message types + * only from these senders: + * + * QUIT, CHANGE_FILE, ACCESS_LOG main + * REMOVE_PID main or a registered prototype + * DATA, APP_RESTART, STATUS the controller + * + * The test sends each type through the handler table of the router from + * each sender: main, the controller, a prototype, a worker, an unknown pid + * and no credential. A refused message must add one to the refusal + * counter, close its two descriptors, and not run the handler. The fixture + * makes a handler that runs crash: nxt_router is NULL, and the reply port + * that the message names is registered. + * + * An accepted message must not change the counter. The test calls the + * check of each type directly. It also sends CHANGE_FILE, DATA, APP_RESTART + * and STATUS through the table: their bodies find no payload or no reply + * port, log that, and return. QUIT, ACCESS_LOG and REMOVE_PID are not sent + * from their sender, because their bodies stop the process or need a + * started router. + * + * A second pass runs with no main port and no controller port, as in a + * restarted router before NEW_PORT. Only a prototype passes, with + * REMOVE_PID. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +#if (NXT_USE_CMSG_PID) + +/* The senders. The first four have a port; NONE sends no credential. */ + +enum { + NXT_ROUTER_SENDER_TEST_MAIN = 0, + NXT_ROUTER_SENDER_TEST_CONTROLLER, + NXT_ROUTER_SENDER_TEST_PROTO, + NXT_ROUTER_SENDER_TEST_WORKER, + NXT_ROUTER_SENDER_TEST_STRANGER, + NXT_ROUTER_SENDER_TEST_NONE, + NXT_ROUTER_SENDER_TEST_PORTS = NXT_ROUTER_SENDER_TEST_STRANGER, + NXT_ROUTER_SENDER_TEST_SENDERS = NXT_ROUTER_SENDER_TEST_NONE + 1 +}; + + +static const char *nxt_router_sender_test_names[] = { + "main", "the controller", "a prototype", "a worker", "a stranger", + "no credential", +}; + + +static const nxt_process_type_t nxt_router_sender_test_port_types[] = { + NXT_PROCESS_MAIN, NXT_PROCESS_CONTROLLER, NXT_PROCESS_PROTOTYPE, + NXT_PROCESS_APP, +}; + + +typedef struct { + const char *name; + nxt_uint_t type; + /* The check; NXT_PROCESS_PROTOTYPE means main or a prototype. */ + nxt_process_type_t check; + /* The body is safe to run from the correct sender. */ + nxt_bool_t run; +} nxt_router_sender_test_type_t; + + +static const nxt_router_sender_test_type_t nxt_router_sender_test_types[] = { + { "QUIT", _NXT_PORT_MSG_QUIT, NXT_PROCESS_MAIN, 0 }, + { "CHANGE_FILE", _NXT_PORT_MSG_CHANGE_FILE, NXT_PROCESS_MAIN, 1 }, + { "ACCESS_LOG", _NXT_PORT_MSG_ACCESS_LOG, NXT_PROCESS_MAIN, 0 }, + { "REMOVE_PID", _NXT_PORT_MSG_REMOVE_PID, NXT_PROCESS_PROTOTYPE, 0 }, + { "DATA", _NXT_PORT_MSG_DATA, NXT_PROCESS_CONTROLLER, 1 }, + { "APP_RESTART", _NXT_PORT_MSG_APP_RESTART, NXT_PROCESS_CONTROLLER, 1 }, + { "STATUS", _NXT_PORT_MSG_STATUS, NXT_PROCESS_CONTROLLER, 1 }, +}; + + +static nxt_pid_t +nxt_router_sender_test_pid(nxt_uint_t sender) +{ + if (sender == NXT_ROUTER_SENDER_TEST_NONE) { + return -1; + } + + return nxt_pid + 1 + sender; +} + + +/* Whether the check accepts the sender; known: the router has both ports. */ + +static nxt_bool_t +nxt_router_sender_test_accepts(nxt_process_type_t check, nxt_uint_t sender, + nxt_bool_t known) +{ + switch (check) { + + case NXT_PROCESS_MAIN: + return (known && sender == NXT_ROUTER_SENDER_TEST_MAIN); + + case NXT_PROCESS_CONTROLLER: + return (known && sender == NXT_ROUTER_SENDER_TEST_CONTROLLER); + + default: + return (sender == NXT_ROUTER_SENDER_TEST_PROTO + || (known && sender == NXT_ROUTER_SENDER_TEST_MAIN)); + } +} + + +/* A message from "sender", with the pid of "claimed" in the header. */ + +static void +nxt_router_sender_test_msg(nxt_port_recv_msg_t *msg, nxt_uint_t type, + nxt_uint_t claimed, nxt_uint_t sender) +{ + nxt_memzero(msg, sizeof(nxt_port_recv_msg_t)); + + msg->fd[0] = -1; + msg->fd[1] = -1; + msg->port_msg.pid = nxt_router_sender_test_pid(claimed); + msg->port_msg.reply_port = 1; + msg->port_msg.type = type; + msg->port_msg.last = 1; + msg->cmsg_pid = nxt_router_sender_test_pid(sender); +} + + +/* As nxt_port_handler() does, through the table of the router. */ + +static void +nxt_router_sender_test_dispatch(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_handler_t *handlers; + + handlers = (nxt_port_handler_t *) nxt_router_process.port_handlers; + + handlers[msg->port_msg.type](task, msg); +} + + +static nxt_int_t +nxt_router_sender_test_refused(nxt_thread_t *thr, nxt_task_t *task, + const nxt_router_sender_test_type_t *t, nxt_uint_t sender) +{ + nxt_fd_t fd[2]; + nxt_uint_t i, refused; + const char *err; + nxt_port_recv_msg_t msg; + + /* + * The header names the controller, so a DATA, APP_RESTART or STATUS + * body that runs finds the reply port and crashes on nxt_router. + */ + nxt_router_sender_test_msg(&msg, t->type, NXT_ROUTER_SENDER_TEST_CONTROLLER, + sender); + + for (i = 0; i < 2; i++) { + fd[i] = open("/dev/null", O_RDONLY); + msg.fd[i] = fd[i]; + } + + err = NULL; + + if (fd[0] == -1 || fd[1] == -1) { + err = "failed to open /dev/null"; + goto done; + } + + refused = nxt_router_test_senders_refused; + + nxt_router_sender_test_dispatch(task, &msg); + + if (nxt_router_test_senders_refused != refused + 1) { + err = "was not refused"; + + } else if (nxt_test_fd_is_open(fd[0]) || nxt_test_fd_is_open(fd[1])) { + err = "left a descriptor open"; + + } else if (msg.fd[0] != -1 || msg.fd[1] != -1) { + err = "left fds in the message"; + } + +done: + + for (i = 0; i < 2; i++) { + if (fd[i] != -1 && nxt_test_fd_is_open(fd[i])) { + (void) close(fd[i]); + } + } + + if (err != NULL) { + nxt_log_alert(thr->log, "router sender test: %s from %s %s", t->name, + nxt_router_sender_test_names[sender], err); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_sender_test_accepted(nxt_thread_t *thr, nxt_task_t *task, + const nxt_router_sender_test_type_t *t, nxt_uint_t sender) +{ + nxt_bool_t res; + nxt_uint_t refused; + const char *who; + nxt_port_recv_msg_t msg; + + who = nxt_router_sender_test_names[sender]; + + refused = nxt_router_test_senders_refused; + + nxt_router_sender_test_msg(&msg, t->type, sender, sender); + + res = nxt_router_test_msg_sender_is(task, &msg, t->check); + + NXT_TEST_CHECK(thr->log, res, "router sender test: %s from %s: check " + "returned 0", t->name, who); + + if (t->run) { + /* The header names a stranger: the body finds no reply port. */ + nxt_router_sender_test_msg(&msg, t->type, + NXT_ROUTER_SENDER_TEST_STRANGER, sender); + + nxt_router_sender_test_dispatch(task, &msg); + } + + NXT_TEST_CHECK(thr->log, nxt_router_test_senders_refused == refused, + "router sender test: %s from %s was refused", t->name, + who); + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_sender_test_run(nxt_thread_t *thr, nxt_task_t *task, + nxt_bool_t known) +{ + nxt_int_t ret; + nxt_uint_t i, s; + + const nxt_router_sender_test_type_t *t; + + for (i = 0; i < nxt_nitems(nxt_router_sender_test_types); i++) { + t = &nxt_router_sender_test_types[i]; + + for (s = 0; s < NXT_ROUTER_SENDER_TEST_SENDERS; s++) { + if (nxt_router_sender_test_accepts(t->check, s, known)) { + ret = nxt_router_sender_test_accepted(thr, task, t, s); + + } else { + ret = nxt_router_sender_test_refused(thr, task, t, s); + } + + if (ret != NXT_OK) { + return ret; + } + } + } + + return NXT_OK; +} + + +static nxt_port_t * +nxt_router_sender_test_port(nxt_task_t *task, nxt_port_id_t id, + nxt_uint_t sender, nxt_process_type_t type) +{ + nxt_port_t *port; + + port = nxt_port_new(task, id, nxt_router_sender_test_pid(sender), type); + + if (nxt_fast_path(port != NULL)) { + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + } + + return port; +} + + +/* A registered process with one port, as the NEW_PORT of main leaves it. */ + +static nxt_port_t * +nxt_router_sender_test_process(nxt_task_t *task, nxt_mp_t *mp, + nxt_uint_t sender, nxt_process_type_t type, nxt_process_t **processp) +{ + nxt_port_t *port; + nxt_process_t *process; + + process = nxt_mp_zalloc(mp, sizeof(nxt_process_t)); + if (nxt_slow_path(process == NULL)) { + return NULL; + } + + process->pid = nxt_router_sender_test_pid(sender); + process->isolated_pid = process->pid; + process->state = NXT_PROCESS_STATE_READY; + nxt_queue_init(&process->ports); + + nxt_runtime_process_add(task, process); + + *processp = process; + + port = nxt_router_sender_test_port(task, 0, sender, type); + if (nxt_slow_path(port == NULL)) { + return NULL; + } + + /* The hold of the fixture on the record. */ + process->use_count = 1; + + nxt_process_port_add(task, process, port); + + return port; +} + +#endif + + +nxt_int_t +nxt_router_sender_test(nxt_thread_t *thr) +{ +#if !(NXT_USE_CMSG_PID) + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "router sender test skipped: no sender credentials"); + return NXT_OK; + +#else + + nxt_mp_t *mp; + nxt_int_t ret; + nxt_uint_t i; + nxt_task_t *task; + nxt_port_t *ports[NXT_ROUTER_SENDER_TEST_PORTS], *reply_port; + nxt_bool_t mutex, hashed; + nxt_router_t *saved_router; + nxt_runtime_t *rt, *saved_rt; + nxt_process_t *processes[NXT_ROUTER_SENDER_TEST_PORTS]; + nxt_event_engine_t engine, *saved_engine; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router sender test started"); + + ret = NXT_ERROR; + reply_port = NULL; + mutex = 0; + hashed = 0; + + nxt_memzero(ports, sizeof(ports)); + nxt_memzero(processes, sizeof(processes)); + + task = thr->task; + task->thread = thr; + + saved_rt = thr->runtime; + saved_engine = thr->engine; + saved_router = nxt_router; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + rt = nxt_mp_zalloc(mp, sizeof(nxt_runtime_t)); + if (nxt_slow_path(rt == NULL)) { + goto done; + } + + rt->mem_pool = mp; + + if (nxt_slow_path(nxt_thread_mutex_create(&rt->processes_mutex) != NXT_OK)) + { + goto done; + } + + mutex = 1; + + /* The handlers run on the main engine only. */ + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + + thr->runtime = rt; + thr->engine = &engine; + rt->main_engine = &engine; + + /* A body that got past its check would dereference this. */ + + nxt_router = NULL; + + for (i = 0; i < NXT_ROUTER_SENDER_TEST_PORTS; i++) { + if (i < NXT_ROUTER_SENDER_TEST_PROTO) { + ports[i] = nxt_router_sender_test_port(task, 0, i, + nxt_router_sender_test_port_types[i]); + + } else { + ports[i] = nxt_router_sender_test_process(task, mp, i, + nxt_router_sender_test_port_types[i], + &processes[i]); + } + + if (nxt_slow_path(ports[i] == NULL)) { + goto done; + } + } + + /* The reply port that a forged DATA, APP_RESTART or STATUS names. */ + + reply_port = nxt_router_sender_test_port(task, 1, + NXT_ROUTER_SENDER_TEST_CONTROLLER, + NXT_PROCESS_CONTROLLER); + + if (nxt_slow_path(reply_port == NULL + || nxt_port_hash_add(&rt->ports, reply_port) != NXT_OK)) + { + goto done; + } + + hashed = 1; + + rt->port_by_type[NXT_PROCESS_MAIN] = ports[NXT_ROUTER_SENDER_TEST_MAIN]; + rt->port_by_type[NXT_PROCESS_CONTROLLER] = + ports[NXT_ROUTER_SENDER_TEST_CONTROLLER]; + + ret = nxt_router_sender_test_run(thr, task, 1); + + if (ret == NXT_OK) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + rt->port_by_type[NXT_PROCESS_CONTROLLER] = NULL; + + ret = nxt_router_sender_test_run(thr, task, 0); + } + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router sender test passed"); + } + +done: + + if (rt != NULL) { + rt->port_by_type[NXT_PROCESS_MAIN] = NULL; + rt->port_by_type[NXT_PROCESS_CONTROLLER] = NULL; + } + + if (hashed) { + (void) nxt_port_hash_remove(&rt->ports, reply_port); + } + + if (reply_port != NULL) { + nxt_port_use(task, reply_port, -1); + } + + for (i = 0; i < NXT_ROUTER_SENDER_TEST_PORTS; i++) { + if (ports[i] != NULL) { + nxt_port_use(task, ports[i], -1); + } + } + + for (i = 0; i < NXT_ROUTER_SENDER_TEST_PORTS; i++) { + if (processes[i] != NULL && processes[i]->registered) { + nxt_runtime_process_remove(rt, processes[i]); + } + } + + nxt_router = saved_router; + thr->engine = saved_engine; + thr->runtime = saved_rt; + + if (mutex) { + nxt_thread_mutex_destroy(&rt->processes_mutex); + } + + nxt_mp_destroy(mp); + + return ret; + +#endif +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 68a50fe88..a83ba2030 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -313,6 +313,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_sender_test(thr) != NXT_OK) { + return 1; + } + if (nxt_router_websocket_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index d20848268..34b69aa76 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -88,6 +88,7 @@ nxt_int_t nxt_router_start_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_app_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_detached_test(nxt_thread_t *thr); +nxt_int_t nxt_router_sender_test(nxt_thread_t *thr); nxt_int_t nxt_router_websocket_test(nxt_thread_t *thr); nxt_int_t nxt_router_prepare_msg_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); From dd3775b5f5e3856735276705cbd98896525944fd Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 19:32:01 +0000 Subject: [PATCH 322/391] refactor(http): share the body buffer allocation and the Host check An HTTP/2 frontend needs two parts of the HTTP/1 request code (ADR 0005, phase 0). This change makes them shared interfaces. The HTTP/1 behaviour does not change. nxt_http_validate_host() was a static helper behind the Host field handler in src/nxt_http_request.c. It cuts the port, drops a trailing dot, lowers upper case into a pool copy and rejects an empty label or a slash. An HTTP/2 frontend checks the :authority pseudo-header with the same rules (phase 0.1). The function is no longer static, and nxt_http.h declares it. The Host handler and its callers do not change. nxt_h1p_request_body_read() allocated the request body buffer inline: a memory buffer when the body fits in body_buffer_size, otherwise an unlinked temporary file under body_temp_path with a memory part of body_buffer_size for the reads. An HTTP/2 frontend needs the same buffer before any DATA frame arrives, sized by Content-Length when it is present, and of unknown length otherwise (phase 0.2). The allocation moves to nxt_http_request_body_alloc(task, r, body_length) in src/nxt_http_request.c, and h1 calls it. body_length is (size_t) -1 when the length is not known. The chunked path passed the same value before, through the content_length_n cast. The function returns NXT_OK or NXT_ERROR. h1 maps NXT_ERROR to 500, as it did for each failure before. The max_body_size limit stays where it was: in the Content-Length handler for a declared length, and in the protocol layer as it writes a chunked body to the file. The memory-copy branch in h1 now bounds the copy by the free space of the body buffer. In that branch this is body_length, as body_buffer_size was before, but the bound now names the buffer that it protects. Unit tests in src/test/ pin both interfaces. The Host test checks the contract of nxt_http_validate_host(). The body test checks the memory or file choice, the sizes and the error path. For the error path, the body test uses a regular file, made with mkstemp() in TMPDIR, as the temporary directory: no file can be made under it on any host. The name is allocated for the length of TMPDIR, and an empty TMPDIR falls back to "/tmp", as an unset one does. Co-Authored-By: Claude Opus 5.5 --- auto/sources | 2 + src/nxt_h1proto.c | 65 +----- src/nxt_http.h | 3 + src/nxt_http_request.c | 80 +++++++- src/test/nxt_http_request_body_alloc_test.c | 208 ++++++++++++++++++++ src/test/nxt_http_validate_host_test.c | 88 +++++++++ src/test/nxt_tests.c | 8 + src/test/nxt_tests.h | 2 + 8 files changed, 397 insertions(+), 59 deletions(-) create mode 100644 src/test/nxt_http_request_body_alloc_test.c create mode 100644 src/test/nxt_http_validate_host_test.c diff --git a/auto/sources b/auto/sources index 36566d6eb..e055d3a7d 100644 --- a/auto/sources +++ b/auto/sources @@ -180,6 +180,8 @@ NXT_TEST_SRCS=" \ src/test/nxt_string_test.c \ src/test/nxt_buf_test.c \ src/test/nxt_http_chunk_parse_test.c \ + src/test/nxt_http_validate_host_test.c \ + src/test/nxt_http_request_body_alloc_test.c \ src/test/nxt_conf_parse_test.c \ src/test/nxt_port_fail_test.c \ src/test/nxt_fd_event_change_test.c \ diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index c6737fdb1..cda6f7e0a 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -930,16 +930,15 @@ nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data) static void nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) { - size_t size, body_length, body_buffer_size, body_rest; + size_t size, body_length, body_rest; ssize_t res; nxt_buf_t *in, *b, *out, *chunk; + nxt_int_t ret; nxt_conn_t *c; nxt_h1proto_t *h1p; nxt_socket_conf_t *skcf; nxt_http_status_t status; - static const nxt_str_t tmp_name_pattern = nxt_string("/req-XXXXXXXX"); - h1p = r->proto.h1; skcf = r->conf->socket_conf; @@ -989,61 +988,13 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) body_length = (size_t) r->content_length_n; - body_buffer_size = nxt_min(skcf->body_buffer_size, body_length); - - if (body_length > body_buffer_size) { - nxt_str_t *tmp_path, tmp_name; - - tmp_path = &skcf->body_temp_path; - - tmp_name.length = tmp_path->length + tmp_name_pattern.length; - - b = nxt_buf_file_alloc(r->mem_pool, - body_buffer_size + sizeof(nxt_file_t) - + tmp_name.length + 1, 0); - if (nxt_slow_path(b == NULL)) { - status = NXT_HTTP_INTERNAL_SERVER_ERROR; - goto error; - } - - tmp_name.start = nxt_pointer_to(b->mem.start, sizeof(nxt_file_t)); - - memcpy(tmp_name.start, tmp_path->start, tmp_path->length); - memcpy(tmp_name.start + tmp_path->length, tmp_name_pattern.start, - tmp_name_pattern.length); - tmp_name.start[tmp_name.length] = '\0'; - - b->file = (nxt_file_t *) b->mem.start; - nxt_memzero(b->file, sizeof(nxt_file_t)); - b->file->fd = -1; - b->file->size = body_length; - - b->mem.start += sizeof(nxt_file_t) + tmp_name.length + 1; - b->mem.pos = b->mem.start; - b->mem.free = b->mem.start; - - b->file->fd = mkstemp((char *) tmp_name.start); - if (nxt_slow_path(b->file->fd == -1)) { - nxt_alert(task, "mkstemp(%s) failed %E", tmp_name.start, nxt_errno); - - status = NXT_HTTP_INTERNAL_SERVER_ERROR; - goto error; - } - - nxt_debug(task, "create body tmp file \"%V\", %d", - &tmp_name, b->file->fd); - - unlink((char *) tmp_name.start); - - } else { - b = nxt_buf_mem_alloc(r->mem_pool, body_buffer_size, 0); - if (nxt_slow_path(b == NULL)) { - status = NXT_HTTP_INTERNAL_SERVER_ERROR; - goto error; - } + ret = nxt_http_request_body_alloc(task, r, body_length); + if (nxt_slow_path(ret != NXT_OK)) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; } - r->body = b; + b = r->body; body_rest = r->chunked ? 1 : body_length; @@ -1102,7 +1053,7 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) } } else { - size = nxt_min(body_buffer_size, size); + size = nxt_min(size, (size_t) nxt_buf_mem_free_size(&b->mem)); b->mem.free = nxt_cpymem(b->mem.free, in->mem.pos, size); in->mem.pos += size; diff --git a/src/nxt_http.h b/src/nxt_http.h index 484bfd9fd..403144e85 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -438,6 +438,8 @@ nxt_http_request_t *nxt_http_request_create(nxt_task_t *task); void nxt_http_request_error(nxt_task_t *task, nxt_http_request_t *r, nxt_http_status_t status); void nxt_http_request_read_body(nxt_task_t *task, nxt_http_request_t *r); +nxt_int_t nxt_http_request_body_alloc(nxt_task_t *task, nxt_http_request_t *r, + size_t body_length); void nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, nxt_work_handler_t body_handler, void *data); void nxt_http_request_ws_frame_start(nxt_task_t *task, nxt_http_request_t *r, @@ -455,6 +457,7 @@ void nxt_http_request_close_handler(nxt_task_t *task, void *obj, void *data); nxt_int_t nxt_http_request_host(void *ctx, nxt_http_field_t *field, uintptr_t data); +nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp); nxt_int_t nxt_http_request_field(void *ctx, nxt_http_field_t *field, uintptr_t offset); nxt_int_t nxt_http_request_content_length(void *ctx, nxt_http_field_t *field, diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 35610f296..ea6af760c 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -9,7 +9,6 @@ #include -static nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp); static void nxt_http_request_start(nxt_task_t *task, void *obj, void *data); static nxt_int_t nxt_http_request_forward(nxt_task_t *task, nxt_http_request_t *r, nxt_http_forward_t *forward); @@ -108,7 +107,7 @@ nxt_http_request_host(void *ctx, nxt_http_field_t *field, uintptr_t data) } -static nxt_int_t +nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp) { u_char *h, ch; @@ -682,6 +681,83 @@ nxt_http_request_read_body(nxt_task_t *task, nxt_http_request_t *r) } +/* + * Allocates r->body for a request body of body_length bytes, or of a + * length not known in advance when body_length is (size_t) -1. A body + * that fits in body_buffer_size is kept in memory; a longer one goes to + * an unlinked temporary file under body_temp_path, with a memory part of + * body_buffer_size for the reads. The protocol layer that fills the + * buffer enforces max_body_size on what it stores. + */ + +nxt_int_t +nxt_http_request_body_alloc(nxt_task_t *task, nxt_http_request_t *r, + size_t body_length) +{ + size_t body_buffer_size; + nxt_buf_t *b; + nxt_socket_conf_t *skcf; + + static const nxt_str_t tmp_name_pattern = nxt_string("/req-XXXXXXXX"); + + skcf = r->conf->socket_conf; + + body_buffer_size = nxt_min(skcf->body_buffer_size, body_length); + + if (body_length > body_buffer_size) { + nxt_str_t *tmp_path, tmp_name; + + tmp_path = &skcf->body_temp_path; + + tmp_name.length = tmp_path->length + tmp_name_pattern.length; + + b = nxt_buf_file_alloc(r->mem_pool, + body_buffer_size + sizeof(nxt_file_t) + + tmp_name.length + 1, 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + tmp_name.start = nxt_pointer_to(b->mem.start, sizeof(nxt_file_t)); + + memcpy(tmp_name.start, tmp_path->start, tmp_path->length); + memcpy(tmp_name.start + tmp_path->length, tmp_name_pattern.start, + tmp_name_pattern.length); + tmp_name.start[tmp_name.length] = '\0'; + + b->file = (nxt_file_t *) b->mem.start; + nxt_memzero(b->file, sizeof(nxt_file_t)); + b->file->fd = -1; + b->file->size = body_length; + + b->mem.start += sizeof(nxt_file_t) + tmp_name.length + 1; + b->mem.pos = b->mem.start; + b->mem.free = b->mem.start; + + b->file->fd = mkstemp((char *) tmp_name.start); + if (nxt_slow_path(b->file->fd == -1)) { + nxt_alert(task, "mkstemp(%s) failed %E", tmp_name.start, nxt_errno); + return NXT_ERROR; + } + + nxt_debug(task, "create body tmp file \"%V\", %d", + &tmp_name, b->file->fd); + + unlink((char *) tmp_name.start); + + } else { + b = nxt_buf_mem_alloc(r->mem_pool, body_buffer_size, 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + } + + r->body = b; + + return NXT_OK; +} + + void nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, nxt_work_handler_t body_handler, void *data) diff --git a/src/test/nxt_http_request_body_alloc_test.c b/src/test/nxt_http_request_body_alloc_test.c new file mode 100644 index 000000000..72cdecdcf --- /dev/null +++ b/src/test/nxt_http_request_body_alloc_test.c @@ -0,0 +1,208 @@ + +/* + * Copyright (C) FreeUnit + */ + +#include +#include +#include +#include "nxt_tests.h" + + +/* + * nxt_http_request_body_alloc() picks the request body buffer from the + * listener's body_buffer_size: a body that fits stays in memory, a longer + * or unknown one gets an unlinked temporary file with a memory part of + * body_buffer_size. The HTTP/1 path used to do this inline; other + * frontends allocate the body the same way, so the table pins the choice + * and the sizes. + */ + +typedef struct { + const char *name; + size_t body_length; + nxt_bool_t file; + size_t mem_size; +} nxt_http_request_body_alloc_test_case_t; + + +static const nxt_http_request_body_alloc_test_case_t + nxt_http_request_body_alloc_test_cases[] = +{ + { "a body below body_buffer_size", 1000, 0, 1000 }, + { "a body of body_buffer_size", 4096, 0, 4096 }, + { "a body above body_buffer_size", 4097, 1, 4096 }, + { "a body of unknown length", (size_t) -1, 1, 4096 }, +}; + + +nxt_int_t +nxt_http_request_body_alloc_test(nxt_thread_t *thr) +{ + nxt_fd_t fd; + const char *tmpdir; + size_t len; + u_char *notdir; + nxt_mp_t *mp; + nxt_buf_t *b; + nxt_int_t ret; + nxt_uint_t i; + nxt_socket_conf_t skcf; + nxt_http_request_t r; + nxt_socket_conf_joint_t joint; + + const nxt_http_request_body_alloc_test_case_t *tc; + + nxt_memzero(&skcf, sizeof(nxt_socket_conf_t)); + nxt_memzero(&joint, sizeof(nxt_socket_conf_joint_t)); + + /* An empty TMPDIR is the same as no TMPDIR. */ + + tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || tmpdir[0] == '\0') { + tmpdir = "/tmp"; + } + + skcf.body_buffer_size = 4096; + skcf.body_temp_path.start = (u_char *) tmpdir; + skcf.body_temp_path.length = nxt_strlen(tmpdir); + + joint.socket_conf = &skcf; + + fd = -1; + + for (i = 0; i < nxt_nitems(nxt_http_request_body_alloc_test_cases); i++) { + tc = &nxt_http_request_body_alloc_test_cases[i]; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + nxt_memzero(&r, sizeof(nxt_http_request_t)); + r.mem_pool = mp; + r.conf = &joint; + + ret = nxt_http_request_body_alloc(thr->task, &r, tc->body_length); + + b = r.body; + + if (ret != NXT_OK || b == NULL) { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "%s: no buffer", tc->name); + goto fail; + } + + if (nxt_buf_is_file(b) != tc->file) { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "%s: file %d expected %d", + tc->name, nxt_buf_is_file(b), tc->file); + goto fail; + } + + if ((size_t) nxt_buf_mem_size(&b->mem) != tc->mem_size + || b->mem.pos != b->mem.start + || b->mem.free != b->mem.start) + { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "%s: memory part %uz expected %uz", + tc->name, nxt_buf_mem_size(&b->mem), tc->mem_size); + goto fail; + } + + if (tc->file) { + fd = b->file->fd; + + if (b->file->fd == -1 || b->file_end != 0 + || b->file->size != (nxt_off_t) tc->body_length) + { + nxt_log_alert(thr->log, "http request body alloc test " + "failed: %s: file fd %d size %O", + tc->name, b->file->fd, b->file->size); + goto fail; + } + + if (!nxt_test_fd_is_open(b->file->fd)) { + nxt_log_alert(thr->log, "http request body alloc test " + "failed: %s: file descriptor is not open", + tc->name); + goto fail; + } + + nxt_fd_close(fd); + fd = -1; + } + + nxt_mp_destroy(mp); + } + + /* + * A temporary path that cannot be opened is an error, not a buffer. + * A regular file is used as the directory: a file under it cannot be + * made on any host (ENOTDIR). + */ + + len = nxt_strlen(tmpdir) + nxt_length("/nxt_body_alloc_XXXXXX") + 1; + + notdir = nxt_malloc(len); + if (notdir == NULL) { + return NXT_ERROR; + } + + (void) nxt_sprintf(notdir, notdir + len, + "%s/nxt_body_alloc_XXXXXX%Z", tmpdir); + + fd = mkstemp((char *) notdir); + if (fd == -1) { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "mkstemp(\"%s\") %E", notdir, nxt_errno); + nxt_free(notdir); + return NXT_ERROR; + } + + nxt_fd_close(fd); + fd = -1; + + skcf.body_temp_path.start = notdir; + skcf.body_temp_path.length = nxt_strlen(notdir); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + (void) unlink((char *) notdir); + nxt_free(notdir); + return NXT_ERROR; + } + + nxt_memzero(&r, sizeof(nxt_http_request_t)); + r.mem_pool = mp; + r.conf = &joint; + + ret = nxt_http_request_body_alloc(thr->task, &r, 8192); + + if (ret != NXT_ERROR || r.body != NULL) { + nxt_log_alert(thr->log, "http request body alloc test failed: " + "a bad temporary path returned %i", ret); + (void) unlink((char *) notdir); + nxt_free(notdir); + goto fail; + } + + nxt_mp_destroy(mp); + (void) unlink((char *) notdir); + nxt_free(notdir); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "http request body alloc test passed"); + + return NXT_OK; + +fail: + + if (fd != -1) { + nxt_fd_close(fd); + } + + nxt_mp_destroy(mp); + + return NXT_ERROR; +} diff --git a/src/test/nxt_http_validate_host_test.c b/src/test/nxt_http_validate_host_test.c new file mode 100644 index 000000000..b93eef3e6 --- /dev/null +++ b/src/test/nxt_http_validate_host_test.c @@ -0,0 +1,88 @@ + +/* + * Copyright (C) FreeUnit + */ + +#include +#include +#include "nxt_tests.h" + + +/* + * nxt_http_validate_host() checks and normalises the value of a Host + * field. It is shared with other frontends, which validate the + * :authority pseudo-header the same way, so the table pins the contract: + * the port is cut off, a trailing dot is dropped, upper case is lowered + * into a pool copy, and an empty label or a slash is a 400. + */ + +nxt_int_t +nxt_http_validate_host_test(nxt_thread_t *thr) +{ + nxt_mp_t *mp; + nxt_str_t host; + nxt_int_t ret; + nxt_uint_t i; + + static const struct { + nxt_str_t input; + nxt_str_t output; + nxt_int_t ret; + } tests[] = { + { nxt_string("example.com"), nxt_string("example.com"), NXT_OK }, + { nxt_string("Example.COM"), nxt_string("example.com"), NXT_OK }, + { nxt_string("example.com."), nxt_string("example.com"), NXT_OK }, + { nxt_string("example.com:8080"), nxt_string("example.com"), NXT_OK }, + { nxt_string("example.com.:8080"), nxt_string("example.com"), NXT_OK }, + { nxt_string("[::1]:8080"), nxt_string("[::1]"), NXT_OK }, + { nxt_string("[::1]"), nxt_string("[::1]"), NXT_OK }, + { nxt_string(""), nxt_string(""), NXT_OK }, + { nxt_string("a..b"), nxt_null_string, + NXT_HTTP_BAD_REQUEST }, + { nxt_string("example.com/path"), nxt_null_string, + NXT_HTTP_BAD_REQUEST }, + { nxt_string("[::1]/"), nxt_null_string, + NXT_HTTP_BAD_REQUEST }, + }; + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + for (i = 0; i < nxt_nitems(tests); i++) { + host = tests[i].input; + + ret = nxt_http_validate_host(&host, mp); + + if (ret != tests[i].ret) { + nxt_log_alert(thr->log, "nxt_http_validate_host(\"%V\") test " + "failed: %i expected %i", + &tests[i].input, ret, tests[i].ret); + goto fail; + } + + if (ret != NXT_OK) { + continue; + } + + if (!nxt_strstr_eq(&host, &tests[i].output)) { + nxt_log_alert(thr->log, "nxt_http_validate_host(\"%V\") test " + "failed: \"%V\" expected \"%V\"", + &tests[i].input, &host, &tests[i].output); + goto fail; + } + } + + nxt_mp_destroy(mp); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "http validate host test passed"); + + return NXT_OK; + +fail: + + nxt_mp_destroy(mp); + + return NXT_ERROR; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index a83ba2030..f7d9f9f98 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -238,6 +238,14 @@ main(int argc, char **argv) return 1; } + if (nxt_http_validate_host_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_http_request_body_alloc_test(thr) != NXT_OK) { + return 1; + } + if (nxt_conf_json_depth_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 34b69aa76..264b00439 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -68,6 +68,8 @@ nxt_int_t nxt_base64_test(nxt_thread_t *thr); nxt_int_t nxt_string_test(nxt_thread_t *thr); nxt_int_t nxt_buf_test(nxt_thread_t *thr); nxt_int_t nxt_http_chunk_parse_test(nxt_thread_t *thr); +nxt_int_t nxt_http_validate_host_test(nxt_thread_t *thr); +nxt_int_t nxt_http_request_body_alloc_test(nxt_thread_t *thr); nxt_int_t nxt_conf_json_depth_test(nxt_thread_t *thr); nxt_int_t nxt_http_route_addr_test(nxt_thread_t *thr); nxt_int_t nxt_conf_map_object_test(nxt_thread_t *thr); From 3a5bb342086257c28d7442fdcdb5be3bed0a7986 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 14:12:39 +0000 Subject: [PATCH 323/391] libunit: run the detached finish retry from an embedder event loop A FINISH report that fails is retried. But the retry ran only from the read loops of libunit: nxt_unit_run_ctx(), nxt_unit_read_buf() and the waits inside them. An application that runs its own event loop never enters those loops. It calls nxt_unit_process_port_msg() once for each readable descriptor. So its failed FINISH was never sent again. The router held the worker detached: out of the idle queues, and counted against "processes": {"max"}. This went on until unrelated traffic arrived, if it ever did. nxt_unit_process_port_msg() now runs a pending retry. It does this at the top of nxt_unit_process_port_msg_impl(), before any code that can return early, and only while the context is online. The return values have these meanings: - NXT_UNIT_OK: call again. The call found no message, but a retry is still pending, and nothing else wakes the embedder. The Node.js and ASGI integrations already call again on NXT_UNIT_OK. - NXT_UNIT_AGAIN: no message, and nothing is pending. The embedder can wait for the next readable event. The call also returns this when the context is offline after the call: a QUIT was processed, or a retry completed a graceful quit that was deferred on the detached state, or the retries gave up. That quit removes the read port, so a receive would wait for a message that the router never sends. - NXT_UNIT_ERROR: the retries gave up. The worker closes. The retry backoff is a deadline, not a wait. The Node.js and ASGI integrations call nxt_unit_process_port_msg() from their only event-loop thread, so a wait inside the call would block that loop for up to 256 ms, about 0.77 s over all ten retries. The context now keeps the next retry time as a monotonic deadline. A call before the deadline skips the retry and only receives. Inside the call the receive polls with a zero timeout. The retries keep the same 2 -> 256 ms steps and the same give-up. The read loops of libunit still wait as before, because they own their thread. If an integration calls again at once on each NXT_UNIT_OK, its event loop spins up to the deadline. So the new function nxt_unit_detached_retry_timeout() gives the delay before the next call, in milliseconds. It returns -1 when no retry is pending or the context is offline. It returns 0 when the retry is due now, or when the last nxt_unit_process_port_msg() call processed a message, because more messages can wait. It returns a value more than 0 only when the last call found no message, and then the value is the time to the deadline. The context keeps a flag for this, and each nxt_unit_process_port_msg() call sets it again. nxt_unit.h documents the function. libunit is a static library with no export list, so the function needs no other change. The Node.js integration starts its uv timer with this delay, and not with zero. Node.js has one port_data_t, with one timer, for each port of the context: the read port and the shared port. Any call runs the retry of the context, whatever the port. So one delayed timer for each context is enough. A delayed timer stays as it is when a later call on either port finds no message. A call that processes a message starts the timer of its port again with zero, and cancels the delayed timer of the other port. The Unit object keeps a list of its port_data_t objects. Unit::quit() stops all of them before it calls nxt_unit_done(), because nxt_unit_done() can free the context, and a timer callback must not call into libunit after that. A stop of a stopped port_data_t does nothing, so the remove_port() calls that libunit makes later are safe. The ASGI integration calls loop.call_later() with the delay, and calls loop.call_soon() as before when the delay is 0 or -1. It keeps the TimerHandle of the delayed call, one for each context. It cancels the handle at the start of each call, so also before the error return of the give-up, and when the context data is freed. After the give-up the loop still runs the lifespan shutdown, and an old handle must not call into libunit then. Any call runs the retry of the context, so one delayed call is enough, and bursts of traffic do not add more timers. The Go integration uses nxt_unit_run_ctx() and nxt_unit_run_shared(), so it needs no change. The other language modules use nxt_unit_run(). nxt_unit_run_shared() runs the same retry. A request from the shared port also runs the application handler, so this loop can end detached work too. The loop now runs on ->online and checks nxt_unit_chk_ready() inside the body, as nxt_unit_run_ctx() does. A loop condition of nxt_unit_chk_ready() would leave a retry behind, because both causes of a false result can occur with a retry pending: the request that ended the detached work can be the one that reached "request_limit", and a graceful quit deferred on the detached state has already cleared ->ready. While requests can still arrive, nxt_unit_shared_port_recv() bounds its wait as nxt_unit_ctx_port_recv() does, so the loop retries instead of blocking in the read. When no request can arrive, there is no descriptor to wait on, so the loop sleeps for the same backoff. This goes on up to the give-up that closes the worker. nxt_unit.h documents the return-code contract for integrations. A QUIT that nxt_unit_process_port_msg() processes runs the quit callback inside the call. The callback of Node.js calls nxt_unit_done(). Then the reference that the call holds is the last one, and the return frees the context. On master the call returned NXT_UNIT_OK after a QUIT, and a caller that used ctx after that touched freed memory. The new function nxt_unit_detached_retry_timeout() reads ctx, so the contract must say when a call is safe. The call now returns NXT_UNIT_AGAIN when the context is offline after the call. The check runs while the call still holds its reference. nxt_unit.h says: after NXT_UNIT_AGAIN or NXT_UNIT_ERROR the caller must not use ctx or port again, and nxt_unit_detached_retry_timeout() may be called only after NXT_UNIT_OK. The Node.js integration keeps its own "stopped" guard. The ASGI integration does not call again on NXT_UNIT_AGAIN, so it no longer schedules a read on a port that the QUIT removed. Tests, in src/test/nxt_unit_port_recv_test.c with the new test hook nxt_unit_test_ctx_read_port(): - nxt_unit_process_port_msg() returns NXT_UNIT_AGAIN when a QUIT runs a quit callback that calls nxt_unit_done(). The test does not use ctx after the call, and the sanitizer catches a use after the free. - nxt_unit_process_port_msg() runs a pending retry, and reports "no message" when that retry completes a deferred quit. - nxt_unit_process_port_msg() returns NXT_UNIT_OK while a retry is pending, no call takes 50 ms or more, the retries span at least 100 ms, and the tenth retry gives up with NXT_UNIT_ERROR. - nxt_unit_detached_retry_timeout() returns -1 with no retry pending, 0 when the first retry is due, a value in (0, 256] after a failed retry, 0 after the deadline, and -1 after the give-up. A loop that waits for this delay reaches the give-up in about 0.77 s with 9 calls. A loop that calls again at once makes about 2 million calls. - nxt_unit_run_shared() runs a pending retry on the way out, both when the retry succeeds and when the retries reach the give-up without a spin. libunit does not link nxt_monotonic_time(), so nxt_unit_detached_now() makes the same choice of clock, in the same order: hg_gethrtime() on HP-UX 11.31, gethrtime() on Solaris and HP-UX, clock_gettime() with CLOCK_MONOTONIC where it exists, mach_absolute_time() scaled with the timebase on macOS, and gettimeofday() elsewhere. A configuration without NXT_HAVE_CLOCK_MONOTONIC builds again. The one difference is that libunit does not use CLOCK_MONOTONIC_COARSE and CLOCK_MONOTONIC_FAST. Those clocks lag up to one kernel tick behind the fine clock of the embedder's timers, so an embedder that waits for the delay from nxt_unit_detached_retry_timeout() would find the retry not due yet, and wake up once more for each retry. The gettimeofday() fallback can step. nxt_unit_detached_delay() treats a deadline that is more than one backoff step ahead as due, so a step delays a retry by at most that step. The test gets its clock from the new test hook nxt_unit_test_detached_now(), so it measures with the clock that paces the retries. When the retries give up, the call quits the context, and the quit can free the port. The ASGI integration now keeps the port id before the call, for its error message, and does not read the port after it. A QUIT that a Node.js call processes runs Unit::quit(), which stops the ports and calls nxt_unit_done(). The context can then be freed before nxt_unit_process_port_msg() returns. So process_port_msg() returns at once when its port is stopped, before it asks for the retry delay. Co-Authored-By: Claude Opus 5.5 --- src/nodejs/unit-http/unit.cpp | 115 ++++++- src/nodejs/unit-http/unit.h | 6 + src/nxt_unit.c | 372 +++++++++++++++++++++- src/nxt_unit.h | 58 ++++ src/python/nxt_python_asgi.c | 115 ++++++- src/python/nxt_python_asgi.h | 2 + src/test/nxt_unit_port_recv_test.c | 479 +++++++++++++++++++++++++++++ 7 files changed, 1126 insertions(+), 21 deletions(-) diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index 7d9395bb6..a55733ae7 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -20,6 +20,7 @@ struct port_data_t { port_data_t(nxt_unit_ctx_t *c, nxt_unit_port_t *p); void process_port_msg(); + void cancel(); void stop(); template @@ -31,10 +32,12 @@ struct port_data_t { nxt_unit_ctx_t *ctx; nxt_unit_port_t *port; + port_data_t *next; uv_poll_t poll; uv_timer_t timer; int ref_count; bool scheduled; + bool delayed; bool stopped; }; @@ -48,7 +51,8 @@ struct req_data_t { port_data_t::port_data_t(nxt_unit_ctx_t *c, nxt_unit_port_t *p) : - ctx(c), port(p), ref_count(0), scheduled(false), stopped(false) + ctx(c), port(p), next(NULL), ref_count(0), scheduled(false), + delayed(false), stopped(false) { timer.type = UV_UNKNOWN_HANDLE; } @@ -57,7 +61,9 @@ port_data_t::port_data_t(nxt_unit_ctx_t *c, nxt_unit_port_t *p) : void port_data_t::process_port_msg() { - int rc, err; + int rc, err, delay; + Unit *obj; + port_data_t *data; rc = nxt_unit_process_port_msg(ctx, port); @@ -65,6 +71,27 @@ port_data_t::process_port_msg() return; } + /* + * A QUIT in this call runs Unit::quit(), which stops all ports and + * calls nxt_unit_done(). The context can be freed now. Do not use it. + * The port data stays until the uv close callbacks run. + */ + + if (stopped) { + return; + } + + /* + * NXT_UNIT_OK also comes when the call found no message and a FINISH + * retry is pending. The retry is not due before a deadline. Wait for + * it with the timer, and do not spin the loop with zero timeouts. + */ + + delay = nxt_unit_detached_retry_timeout(ctx); + if (delay < 0) { + delay = 0; + } + if (timer.type == UV_UNKNOWN_HANDLE) { err = uv_timer_init(poll.loop, &timer); if (err < 0) { @@ -76,19 +103,79 @@ port_data_t::process_port_msg() timer.data = this; } - if (!scheduled && !stopped) { - uv_timer_start(&timer, timer_callback, 0, 0); + /* + * Any call runs the retry of the context, whatever the port. So one + * delayed call for each context is enough. A delayed call is pending + * on this port or on the other port of the context. Keep it, unless + * this call processed a message. Then more messages can wait, so + * call again at once, and cancel the delayed call of the other port. + */ + + obj = reinterpret_cast(ctx->unit->data); + + if (delay > 0) { + if (scheduled) { + return; + } + + for (data = obj->ports_; data != NULL; data = data->next) { + if (data->delayed) { + return; + } + } + + } else { + if (scheduled && !delayed) { + return; + } - scheduled = true; + for (data = obj->ports_; data != NULL; data = data->next) { + if (data != this && data->delayed) { + data->cancel(); + } + } } + + uv_timer_start(&timer, timer_callback, delay, 0); + + scheduled = true; + delayed = (delay > 0); +} + + +void +port_data_t::cancel() +{ + uv_timer_stop(&timer); + + scheduled = false; + delayed = false; } void port_data_t::stop() { + Unit *obj; + port_data_t **link; + + if (stopped) { + return; + } + stopped = true; + /* The context can be freed after this. Unlink from its ports. */ + + obj = reinterpret_cast(ctx->unit->data); + + for (link = &obj->ports_; *link != NULL; link = &(*link)->next) { + if (*link == this) { + *link = next; + break; + } + } + uv_poll_stop(&poll); uv_close((uv_handle_t *) &poll, delete_data); @@ -126,6 +213,8 @@ port_data_t::timer_callback(uv_timer_t *handle) data = get(handle); data->scheduled = false; + data->delayed = false; + if (data->stopped) { return; } @@ -150,7 +239,8 @@ port_data_t::delete_data(uv_handle_t* handle) Unit::Unit(napi_env env, napi_value jsthis): nxt_napi(env), wrapper_(wrap(jsthis, this, destroy)), - unit_ctx_(nullptr) + unit_ctx_(nullptr), + ports_(nullptr) { nxt_unit_debug(NULL, "Unit::Unit()"); } @@ -512,6 +602,9 @@ Unit::add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) data->ref_count++; data->poll.data = data; + + data->next = obj->ports_; + obj->ports_ = data; } return NXT_UNIT_OK; @@ -563,6 +656,16 @@ Unit::quit(nxt_unit_ctx_t *ctx) nxt_unit_debug(ctx, "quit: %s", e.str); } + /* + * nxt_unit_done() can free the context. No timer of its ports may + * call into libunit after that. Stop the ports that libunit did not + * remove yet. A later remove_port() call finds them stopped. + */ + + while (ports_ != NULL) { + ports_->stop(); + } + nxt_unit_done(ctx); } diff --git a/src/nodejs/unit-http/unit.h b/src/nodejs/unit-http/unit.h index 1aa93073a..1ce2c6534 100644 --- a/src/nodejs/unit-http/unit.h +++ b/src/nodejs/unit-http/unit.h @@ -9,11 +9,16 @@ #include "nxt_napi.h" +struct port_data_t; + + class Unit : public nxt_napi { public: static napi_value init(napi_env env, napi_value exports); private: + friend struct port_data_t; + Unit(napi_env env, napi_value jsthis); ~Unit(); @@ -81,6 +86,7 @@ class Unit : public nxt_napi { napi_ref wrapper_; nxt_unit_ctx_t *unit_ctx_; + port_data_t *ports_; }; diff --git a/src/nxt_unit.c b/src/nxt_unit.c index a2fde4dbc..cc2a64b5b 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -84,6 +84,11 @@ static void nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx); static int nxt_unit_ctx_detached_retry(nxt_unit_ctx_t *ctx); static int nxt_unit_detached_timeout(nxt_unit_ctx_impl_t *ctx_impl); static int nxt_unit_detached_poll(nxt_unit_ctx_t *ctx, int fd); +static void nxt_unit_detached_sleep(nxt_unit_ctx_impl_t *ctx_impl); +static uint64_t nxt_unit_detached_now(void); +static uint64_t nxt_unit_detached_delay(nxt_unit_ctx_impl_t *ctx_impl, + uint64_t now); +static int nxt_unit_detached_wake(nxt_unit_ctx_impl_t *ctx_impl); static int nxt_unit_send_detached(nxt_unit_ctx_t *ctx, uint8_t state); static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); @@ -435,6 +440,24 @@ struct nxt_unit_ctx_impl_s { /* Failed FINISH sends so far; 0 when none is pending. */ uint8_t detached_retries; + /* + * Set while nxt_unit_process_port_msg() runs. The embedder owns the + * event loop, so a pending retry must not block it. + */ + uint8_t detached_nowait; /* 1 bit */ + + /* + * The last nxt_unit_process_port_msg() call found no message, and it + * returned NXT_UNIT_OK only for a pending FINISH retry. + */ + uint8_t detached_idle; /* 1 bit */ + + /* + * The earliest time for the next FINISH retry from + * nxt_unit_process_port_msg(), in milliseconds of a monotonic clock. + */ + uint64_t detached_deadline; + /* * The START edge was never delivered. The worker retires when the * request handler returns. @@ -824,6 +847,9 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, ctx_impl->detached = NXT_UNIT_DETACHED_NONE; ctx_impl->detached_retries = 0; + ctx_impl->detached_nowait = 0; + ctx_impl->detached_idle = 0; + ctx_impl->detached_deadline = 0; ctx_impl->detached_unreported = 0; nxt_queue_init(&ctx_impl->free_req); @@ -3866,6 +3892,7 @@ nxt_unit_ctx_detached_done(nxt_unit_ctx_t *ctx) if (ctx_impl->detached_retries == 0) { ctx_impl->detached_retries = 1; + ctx_impl->detached_deadline = 0; } } @@ -3952,13 +3979,15 @@ nxt_unit_detached_timeout(nxt_unit_ctx_impl_t *ctx_impl) /* * Wait for "fd" to become readable while a FINISH retry is pending. * Returns NXT_UNIT_AGAIN when the wait expires, so that the caller returns - * to the read loop, which retries. + * to the read loop, which retries. Inside nxt_unit_process_port_msg() the + * wait is zero. The embedder's event loop must not block, so the backoff + * is a deadline there. */ static int nxt_unit_detached_poll(nxt_unit_ctx_t *ctx, int fd) { - int nevents; + int nevents, timeout; struct pollfd pfd; nxt_unit_ctx_impl_t *ctx_impl; @@ -3968,7 +3997,10 @@ nxt_unit_detached_poll(nxt_unit_ctx_t *ctx, int fd) pfd.events = POLLIN; pfd.revents = 0; - nevents = poll(&pfd, 1, nxt_unit_detached_timeout(ctx_impl)); + timeout = ctx_impl->detached_nowait ? 0 + : nxt_unit_detached_timeout(ctx_impl); + + nevents = poll(&pfd, 1, timeout); if (nevents == 0 || (nevents == -1 && errno == EINTR)) { return NXT_UNIT_AGAIN; @@ -3978,6 +4010,174 @@ nxt_unit_detached_poll(nxt_unit_ctx_t *ctx, int fd) } +/* + * Sleep for the backoff before the next FINISH retry. This is for a caller + * that has no descriptor to wait on. nxt_unit_detached_poll() does the + * same for a loop that has one. + */ + +static void +nxt_unit_detached_sleep(nxt_unit_ctx_impl_t *ctx_impl) +{ + struct timespec ts; + + ts.tv_sec = 0; + ts.tv_nsec = nxt_unit_detached_timeout(ctx_impl) * 1000000L; + + (void) nanosleep(&ts, NULL); +} + + +/* + * The time for the FINISH retry deadline, in milliseconds. libunit does + * not link nxt_monotonic_time(), so this makes the same choice of clock, + * in the same order, with one difference. nxt_monotonic_time() prefers + * CLOCK_MONOTONIC_COARSE on Linux and CLOCK_MONOTONIC_FAST on FreeBSD. + * Those clocks lag up to one kernel tick behind the fine clock of the + * embedder's timers. An embedder that waits for the delay from + * nxt_unit_detached_retry_timeout() would then find the retry not due + * yet, and wake up once more for each retry. So the fine clock is used. + * See src/nxt_time.c for the other choices. + */ + +static uint64_t +nxt_unit_detached_now(void) +{ +#if (NXT_HAVE_HG_GETHRTIME) + + return (uint64_t) hg_gethrtime() / 1000000; + +#elif (NXT_SOLARIS || NXT_HPUX) + + return (uint64_t) gethrtime() / 1000000; + +#elif (NXT_HAVE_CLOCK_MONOTONIC) + + struct timespec ts; + + (void) clock_gettime(CLOCK_MONOTONIC, &ts); + + return (uint64_t) ts.tv_sec * 1000 + ts.tv_nsec / 1000000; + +#elif (NXT_MACOSX) + + /* + * mach_absolute_time() returns ticks. The timebase gives their length + * in nanoseconds as a fraction: 1/1 on Intel Macs, 125/3 on Apple + * silicon. The quotient and the remainder are scaled apart, so the + * result is exact and no product can overflow before the nanoseconds + * themselves do, in centuries. + */ + + uint64_t ns, ticks; + mach_timebase_info_data_t tb; + + (void) mach_timebase_info(&tb); + + ticks = mach_absolute_time(); + + ns = ticks / tb.denom * tb.numer + ticks % tb.denom * tb.numer / tb.denom; + + return ns / 1000000; + +#else + + /* + * No monotonic clock. nxt_monotonic_time() emulates one with a + * per-thread state. This helper keeps no state, so a step of the + * clock moves the deadline. nxt_unit_detached_delay() bounds that. + */ + + struct timeval tv; + + (void) gettimeofday(&tv, NULL); + + return (uint64_t) tv.tv_sec * 1000 + tv.tv_usec / 1000; + +#endif +} + + +/* + * The time from "now" to the FINISH retry deadline, in milliseconds, or 0 + * when the retry is due. A deadline that is more than one backoff step + * ahead can only come from a clock that went back. The retry is due then: + * an early retry is harmless, and a late one holds the worker detached. + */ + +static uint64_t +nxt_unit_detached_delay(nxt_unit_ctx_impl_t *ctx_impl, uint64_t now) +{ + uint64_t delay; + + if (now >= ctx_impl->detached_deadline) { + return 0; + } + + delay = ctx_impl->detached_deadline - now; + + if (nxt_slow_path(delay > (uint64_t) nxt_unit_detached_timeout(ctx_impl))) { + return 0; + } + + return delay; +} + + +/* + * Return the code for a call that found no message. A pending FINISH + * retry needs another call, and the embedder's descriptor stays quiet + * until unrelated traffic arrives. So report NXT_UNIT_OK: an integration + * that drives its own event loop calls again on that code, and + * NXT_UNIT_AGAIN stops it. Nothing waits here. The backoff is + * ->detached_deadline, and a call before the deadline only receives. + */ + +static int +nxt_unit_detached_wake(nxt_unit_ctx_impl_t *ctx_impl) +{ + if (nxt_fast_path(ctx_impl->detached_retries == 0 || !ctx_impl->online)) { + return NXT_UNIT_AGAIN; + } + + ctx_impl->detached_idle = 1; + + return NXT_UNIT_OK; +} + + +/* + * Tell an integration with its own event loop when to call + * nxt_unit_process_port_msg() again. A delay is returned only after a + * call that found no message. After a call that processed a message, + * more messages can wait, so the next call must come at once. + */ + +int +nxt_unit_detached_retry_timeout(nxt_unit_ctx_t *ctx) +{ + uint64_t now; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + if (ctx_impl->detached_retries == 0 + || ctx_impl->detached_retries > 10 + || !ctx_impl->online) + { + return -1; + } + + if (!ctx_impl->detached_idle) { + return 0; + } + + now = nxt_unit_detached_now(); + + return (int) nxt_min(nxt_unit_detached_delay(ctx_impl, now), INT_MAX); +} + + #if (NXT_TESTS) uint8_t nxt_unit_test_ctx_detached(nxt_unit_ctx_t *ctx) @@ -4055,6 +4255,24 @@ nxt_unit_test_ctx_detached_retry(nxt_unit_ctx_t *ctx) } +nxt_unit_port_t * +nxt_unit_test_ctx_read_port(nxt_unit_ctx_t *ctx) +{ + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + return ctx_impl->read_port; +} + + +uint64_t +nxt_unit_test_detached_now(void) +{ + return nxt_unit_detached_now(); +} + + uint8_t nxt_unit_test_ctx_online(nxt_unit_ctx_t *ctx) { @@ -6076,14 +6294,63 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) int rc; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_ctx_use(ctx); lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); rc = NXT_UNIT_OK; - while (nxt_fast_path(nxt_unit_chk_ready(ctx))) { + while (nxt_fast_path(ctx_impl->online)) { + + /* + * A request from the shared port also runs the application's + * handler. So this loop can end detached work, and the finish + * report can fail. Retry it here, as nxt_unit_run_ctx() does. The + * wait below is bounded while a retry is pending. + */ + + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + rc = nxt_unit_ctx_detached_retry(ctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + break; + } + + /* + * The retry can complete a deferred graceful quit. That + * removes the read port. Leave with the retry's NXT_UNIT_OK. + */ + + if (nxt_slow_path(!ctx_impl->online)) { + break; + } + } + + if (nxt_slow_path(!nxt_unit_chk_ready(ctx))) { + + /* + * No request can arrive now, so the loop ends here. The loop + * runs on ->online and not on nxt_unit_chk_ready() for one + * reason: a pending retry must not be left behind here. Both + * causes of a false nxt_unit_chk_ready() can occur with a retry + * pending. The request that ended the detached work can be the + * one that reached "request_limit". A graceful quit deferred on + * the detached state has already cleared ->ready. There is no + * descriptor to wait on, so sleep for the backoff and retry + * above. This goes on up to the give-up that closes the worker. + */ + + if (nxt_fast_path(ctx_impl->detached_retries == 0)) { + break; + } + + nxt_unit_detached_sleep(ctx_impl); + + continue; + } + rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { rc = NXT_UNIT_ERROR; @@ -6094,6 +6361,11 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) rc = nxt_unit_shared_port_recv(ctx, lib->shared_port, rbuf); if (rc == NXT_UNIT_AGAIN) { + if (nxt_slow_path(ctx_impl->detached_retries > 0)) { + nxt_unit_read_buf_release(ctx, rbuf); + continue; + } + goto retry; } @@ -6158,10 +6430,31 @@ nxt_unit_process_port_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { int rc; + nxt_unit_ctx_impl_t *ctx_impl; + + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + nxt_unit_ctx_use(ctx); + ctx_impl->detached_nowait = 1; + ctx_impl->detached_idle = 0; + rc = nxt_unit_process_port_msg_impl(ctx, port); + ctx_impl->detached_nowait = 0; + + /* + * A QUIT in this call runs the quit callback. When the callback calls + * nxt_unit_done(), the reference above is the last one, and the release + * below frees the context. The caller cannot know this from ctx, so + * tell it here: NXT_UNIT_AGAIN means "nothing to schedule, stop". An + * offline context has no read port, so a call again has no use. + */ + + if (rc == NXT_UNIT_OK && nxt_slow_path(!ctx_impl->online)) { + rc = NXT_UNIT_AGAIN; + } + nxt_unit_ctx_release(ctx); return rc; @@ -6172,13 +6465,58 @@ static int nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { int rc; + uint64_t now; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + /* + * The embedder runs its own event loop, so this call is the only + * wake-up that libunit gets. The read loops retry from their own + * waits. Run the retry before anything that can return early. If + * not, a worker whose finish report failed stays detached in the + * router until traffic arrives. The backoff is a deadline and not a + * wait, because a wait would block the embedder's loop. A call before + * the deadline only receives. + */ + + if (nxt_slow_path(ctx_impl->detached_retries > 0 && ctx_impl->online)) { + now = nxt_unit_detached_now(); + + if (nxt_unit_detached_delay(ctx_impl, now) > 0) { + goto recv; + } + + rc = nxt_unit_ctx_detached_retry(ctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + return rc; + } + + if (ctx_impl->detached_retries > 0) { + ctx_impl->detached_deadline = now + + nxt_unit_detached_timeout(ctx_impl); + } + + /* + * The retry can complete a graceful quit that was deferred on the + * detached state. That removes the read port. Report "no + * message" here. A receive would wait for a message that the + * router will never send, and NXT_UNIT_AGAIN makes the embedder + * stop calling. + */ + + if (nxt_slow_path(!ctx_impl->online)) { + return NXT_UNIT_AGAIN; + } + } + +recv: if (port == lib->shared_port && !nxt_unit_chk_ready(ctx)) { - return NXT_UNIT_AGAIN; + return nxt_unit_detached_wake(ctx_impl); } rbuf = nxt_unit_read_buf_get(ctx); @@ -6195,6 +6533,11 @@ nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) if (rc != NXT_UNIT_OK) { nxt_unit_read_buf_release(ctx, rbuf); + + if (rc == NXT_UNIT_AGAIN) { + return nxt_unit_detached_wake(ctx_impl); + } + return rc; } @@ -7384,8 +7727,10 @@ nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf) { int res; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_port_impl_t *port_impl; + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); retry: @@ -7397,6 +7742,23 @@ nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, } if (res == NXT_UNIT_AGAIN) { + + /* + * Bound the wait while a detached finish retry is pending, as + * nxt_unit_ctx_port_recv() does. Then the caller returns to its + * loop and retries. Without this, the caller blocks here until a + * request comes. + */ + + if (nxt_slow_path(ctx_impl->detached_retries > 0 + && port->in_fd != -1)) + { + res = nxt_unit_detached_poll(ctx, port->in_fd); + if (res != NXT_UNIT_OK) { + return res; + } + } + res = nxt_unit_port_recv(ctx, port, rbuf); if (nxt_slow_path(res == NXT_UNIT_ERROR)) { return NXT_UNIT_ERROR; diff --git a/src/nxt_unit.h b/src/nxt_unit.h index 5ce647a37..a622174e7 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -228,8 +228,51 @@ nxt_unit_request_info_t *nxt_unit_dequeue_request(nxt_unit_ctx_t *ctx); */ int nxt_unit_run_once(nxt_unit_ctx_t *ctx); +/* + * Process one message from "port", for an integration that drives its own + * event loop. The call does not wait for a message. It returns: + * + * - NXT_UNIT_OK: call again. The call processed a message, or it found + * no message and a FINISH retry is pending. Get the delay before the + * next call from nxt_unit_detached_retry_timeout(). + * - NXT_UNIT_AGAIN: do not call again until the port becomes readable. + * The call found no message and no retry is pending, or the context is + * offline after the call: a QUIT was processed, or a retry completed a + * deferred graceful quit, or the retries gave up. + * - NXT_UNIT_ERROR: the worker is closing. Do not call again. + * + * A QUIT runs the quit callback inside the call. When that callback calls + * nxt_unit_done(), the context can be freed when the call returns. So + * after NXT_UNIT_AGAIN or NXT_UNIT_ERROR the caller must not use ctx or + * port again. Only after NXT_UNIT_OK may it call + * nxt_unit_detached_retry_timeout(). + */ int nxt_unit_process_port_msg(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); +/* + * Get the delay before the next nxt_unit_process_port_msg() call, in + * milliseconds, for an integration that drives its own event loop. Use it + * only after nxt_unit_process_port_msg() returns NXT_UNIT_OK. After + * NXT_UNIT_AGAIN or NXT_UNIT_ERROR the context can be freed already, see + * nxt_unit_process_port_msg(). NXT_UNIT_OK means "call again". Usually + * the call processed a message and more can wait. But the call also + * returns NXT_UNIT_OK when it found no message and a FINISH retry is + * pending (see nxt_unit_request_done_detached()). Then the next retry is + * not due until a deadline, and a call before it only receives. + * + * The function returns: + * + * - -1: no retry is pending, or the context is offline. Call again at + * once, as for any NXT_UNIT_OK. + * - 0: call again at once. The retry is due now, or the last + * nxt_unit_process_port_msg() call on the context processed a message. + * - more than 0: the last call found no message. Call again after this + * many milliseconds, or earlier when the port becomes readable. + * + * The value is not more than INT_MAX. + */ +int nxt_unit_detached_retry_timeout(nxt_unit_ctx_t *ctx); + /* Destroy application library object. */ void nxt_unit_done(nxt_unit_ctx_t *); @@ -331,6 +374,19 @@ void nxt_unit_request_done(nxt_unit_request_info_t *req, int rc); * may run detached work at once: the worker stays busy until the last of * them returns. PHP's fastcgi_finish_request() is the caller this exists * for, and PHP runs one context. + * + * The report to the router can fail. Then libunit retries it from its + * read loops and from nxt_unit_process_port_msg(). The router holds the + * worker busy until a retry succeeds. In that time nothing wakes an + * integration that drives its own event loop. So while a retry is + * pending, nxt_unit_process_port_msg() returns NXT_UNIT_OK, and not + * NXT_UNIT_AGAIN, to ask for the next call. The call never waits for the + * retry backoff, because a wait would block the event loop. It keeps a + * deadline instead, and a call before the deadline only receives. An + * integration that calls again on NXT_UNIT_OK must not call at once each + * time, or its loop spins until the deadline. It gets the delay for the + * next call from nxt_unit_detached_retry_timeout(). The retries give up + * after about 0.8 s and close the worker. */ void nxt_unit_request_done_detached(nxt_unit_request_info_t *req, int rc); @@ -424,6 +480,8 @@ uint8_t nxt_unit_test_ctx_detached_unreported(nxt_unit_ctx_t *ctx); void nxt_unit_test_ctx_detached_start(nxt_unit_ctx_t *ctx); void nxt_unit_test_ctx_detached_done(nxt_unit_ctx_t *ctx); int nxt_unit_test_ctx_detached_retry(nxt_unit_ctx_t *ctx); +nxt_unit_port_t *nxt_unit_test_ctx_read_port(nxt_unit_ctx_t *ctx); +uint64_t nxt_unit_test_detached_now(void); uint8_t nxt_unit_test_ctx_online(nxt_unit_ctx_t *ctx); uint8_t nxt_unit_test_ctx_ready(nxt_unit_ctx_t *ctx); void nxt_unit_test_ctx_set_ready(nxt_unit_ctx_t *ctx, uint8_t val); diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index a123e9411..5cd145957 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -46,6 +46,8 @@ static void nxt_py_asgi_quit(nxt_unit_ctx_t *ctx); static void nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx); static PyObject *nxt_py_asgi_port_read(PyObject *self, PyObject *args); +static void nxt_py_asgi_retry_timer_cancel(nxt_unit_ctx_t *ctx, + nxt_py_asgi_ctx_data_t *ctx_data); static void nxt_python_asgi_done(void); static PyObject *nxt_py_port_read; @@ -269,6 +271,7 @@ nxt_python_asgi_ctx_data_alloc(void **pdata, int main) { "add_reader", &ctx_data->loop_add_reader }, { "remove_reader", &ctx_data->loop_remove_reader }, { "call_soon", &ctx_data->loop_call_soon }, + { "call_later", &ctx_data->loop_call_later }, { "run_until_complete", &ctx_data->loop_run_until_complete }, { "create_future", &ctx_data->loop_create_future }, }; @@ -366,10 +369,15 @@ nxt_python_asgi_ctx_data_free(void *data) ctx_data = data; + /* The delayed call keeps the pointer of the context. It must not run. */ + + nxt_py_asgi_retry_timer_cancel(NULL, ctx_data); + Py_XDECREF(ctx_data->loop_run_until_complete); Py_XDECREF(ctx_data->loop_create_future); Py_XDECREF(ctx_data->loop_create_task); Py_XDECREF(ctx_data->loop_call_soon); + Py_XDECREF(ctx_data->loop_call_later); Py_XDECREF(ctx_data->loop_add_reader); Py_XDECREF(ctx_data->loop_remove_reader); Py_XDECREF(ctx_data->quit_future); @@ -1162,8 +1170,8 @@ nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx) static PyObject * nxt_py_asgi_port_read(PyObject *self, PyObject *args) { - int rc; - PyObject *arg0, *arg1, *res; + int rc, delay, port_id; + PyObject *arg0, *arg1, *res, *timeout; Py_ssize_t n; nxt_unit_ctx_t *ctx; nxt_unit_port_t *port; @@ -1195,24 +1203,82 @@ nxt_py_asgi_port_read(PyObject *self, PyObject *args) port = PyLong_AsVoidPtr(arg1); + /* + * When the FINISH retries give up, the call quits the context and can + * free the port. Keep its id for the error message. + */ + + port_id = port->id.id; + rc = nxt_unit_process_port_msg(ctx, port); nxt_unit_debug(ctx, "asgi_port_read(%p,%p): %d", ctx, port, rc); + ctx_data = ctx->data; + + /* + * Each call can run the pending FINISH retry of the context, whatever + * the port. So one delayed call for each context is enough. Cancel + * the old one before a new call is scheduled. Otherwise each burst of + * traffic adds one more timer, and all of them wake the loop. Cancel + * it also before the error return. After the give-up the loop still + * runs the lifespan shutdown, and the old call must not run then. + */ + + nxt_py_asgi_retry_timer_cancel(ctx, ctx_data); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return PyErr_Format(PyExc_RuntimeError, - "error processing port %d message", port->id.id); + "error processing port %d message", port_id); } if (rc == NXT_UNIT_OK) { - ctx_data = ctx->data; - res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_soon, - nxt_py_port_read, - arg0, arg1, NULL); - if (nxt_slow_path(res == NULL)) { - nxt_unit_alert(ctx, "Python failed to call 'loop.call_soon'"); - nxt_python_print_exception(); + /* + * NXT_UNIT_OK also comes when the call found no message and a + * FINISH retry is pending. The retry is not due before a deadline. + * Call again at that deadline, and do not spin the loop with + * call_soon(). + */ + + delay = nxt_unit_detached_retry_timeout(ctx); + + if (delay > 0) { + timeout = PyFloat_FromDouble(delay / 1000.0); + if (nxt_slow_path(timeout == NULL)) { + nxt_unit_alert(ctx, "Python failed to create a float"); + nxt_python_print_exception(); + + Py_RETURN_NONE; + } + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_later, + timeout, nxt_py_port_read, + arg0, arg1, NULL); + + Py_DECREF(timeout); + + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to call 'loop.call_later'"); + nxt_python_print_exception(); + + Py_RETURN_NONE; + } + + /* Keep the TimerHandle, so that a later call can cancel it. */ + + ctx_data->retry_timer = res; + + Py_RETURN_NONE; + + } else { + res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_soon, + nxt_py_port_read, + arg0, arg1, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to call 'loop.call_soon'"); + nxt_python_print_exception(); + } } Py_XDECREF(res); @@ -1222,6 +1288,35 @@ nxt_py_asgi_port_read(PyObject *self, PyObject *args) } +static void +nxt_py_asgi_retry_timer_cancel(nxt_unit_ctx_t *ctx, + nxt_py_asgi_ctx_data_t *ctx_data) +{ + PyObject *timer, *res; + + timer = ctx_data->retry_timer; + + if (timer == NULL) { + return; + } + + ctx_data->retry_timer = NULL; + + /* Cancel of a TimerHandle that already ran does nothing. */ + + res = PyObject_CallMethod(timer, "cancel", NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to cancel the retry timer"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + } + + Py_DECREF(timer); +} + + PyObject * nxt_py_asgi_enum_headers(PyObject *headers, nxt_py_asgi_enum_header_cb cb, void *data) diff --git a/src/python/nxt_python_asgi.h b/src/python/nxt_python_asgi.h index 94478a361..a9783536c 100644 --- a/src/python/nxt_python_asgi.h +++ b/src/python/nxt_python_asgi.h @@ -29,6 +29,8 @@ typedef struct { PyObject *loop_create_future; PyObject *loop_create_task; PyObject *loop_call_soon; + PyObject *loop_call_later; + PyObject *retry_timer; PyObject *loop_add_reader; PyObject *loop_remove_reader; PyObject *quit_future; diff --git a/src/test/nxt_unit_port_recv_test.c b/src/test/nxt_unit_port_recv_test.c index 1b4e66c4a..6870498c8 100644 --- a/src/test/nxt_unit_port_recv_test.c +++ b/src/test/nxt_unit_port_recv_test.c @@ -53,6 +53,8 @@ static int nxt_port_recv_test_failures; static int nxt_port_recv_test_step; static int nxt_port_recv_test_block; +static int nxt_port_recv_test_quit_done; +static int nxt_port_recv_test_quit_calls; static int nxt_port_recv_test_shared_queue_fd = -1; static int nxt_port_recv_test_sock_fd = -1; static int nxt_port_recv_test_queue_fd = -1; @@ -86,6 +88,21 @@ nxt_port_recv_test_handler(nxt_unit_request_info_t *req) } +/* + * The quit callback of an embedder such as Node.js releases the context + * at once. That can free it when the libunit call returns. + */ +static void +nxt_port_recv_test_quit(nxt_unit_ctx_t *ctx) +{ + nxt_port_recv_test_quit_calls++; + + if (nxt_port_recv_test_quit_done) { + nxt_unit_done(ctx); + } +} + + /* * Captures the read port queue of the context being announced to the * router. libunit closes that descriptor as soon as the announcement is @@ -604,6 +621,378 @@ nxt_port_recv_test_deferred_quit_read(nxt_unit_ctx_t *ctx, int run_ctx, } +static void +nxt_port_recv_test_deferred_quit_port_msg(nxt_unit_ctx_t *ctx, + const char *name) +{ + int rc; + pid_t pid; + + /* + * An embedder that drives its own event loop never enters a libunit + * read loop. So nxt_unit_process_port_msg() is the only wake-up that + * can run the retry. Without it, the FINISH edge is never sent again, + * and the router holds the worker detached. Here the retry completes + * a deferred graceful quit, which removes the read port. So the call + * must also report "no message" and must not receive. The alarm + * catches a wait for a message that the router will never send. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + nxt_unit_test_ctx_quit_graceful(ctx); + + /* One failure: the handler return arms the retry, the retry sends. */ + nxt_unit_test_send_detached_failures(1); + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached(ctx) != 1 + || nxt_unit_test_ctx_detached_retries(ctx) != 1) + { + _exit(1); + } + + alarm(5); + + rc = nxt_unit_process_port_msg(ctx, + nxt_unit_test_ctx_read_port(ctx)); + + if (rc != NXT_UNIT_AGAIN + || nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 0) + { + _exit(2); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +/* + * The clock of nxt_unit_detached_now(), in milliseconds, through a test + * hook. The retries are paced with a deadline on that clock, so the + * test measures them with the same clock. + */ + +static long +nxt_port_recv_test_ms(uint64_t from, uint64_t to) +{ + return (long) (to - from); +} + + +static void +nxt_port_recv_test_port_msg_retry_pending(nxt_unit_ctx_t *ctx, + const char *name) +{ + int rc; + long call_ms, max_ms; + pid_t pid; + uint64_t end, start, t0, t1; + unsigned int calls; + + /* + * A retry that fails again leaves the context online with the FINISH + * still pending. Nothing else wakes the embedder: its read port stays + * quiet until unrelated traffic arrives. The call must report + * NXT_UNIT_OK, because an embedder calls again on that code. It must + * not report NXT_UNIT_AGAIN, which stops the embedder and leaves the + * worker detached. The call must not wait for the backoff either, + * because that would freeze the embedder's event loop. Each call + * returns at once, and a deadline paces the retries. Reads block, so + * the alarm catches a wait for a message. The slowest call catches a + * bounded wait. The elapsed time catches retries that spin through the + * budget instead of pacing it. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + nxt_unit_test_send_detached_failures(20); + + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached_retries(ctx) != 1) { + _exit(1); + } + + alarm(10); + + max_ms = 0; + + start = nxt_unit_test_detached_now(); + + for (calls = 1; calls <= 10000000; calls++) { + t0 = nxt_unit_test_detached_now(); + + rc = nxt_unit_process_port_msg(ctx, + nxt_unit_test_ctx_read_port(ctx)); + + t1 = nxt_unit_test_detached_now(); + + call_ms = nxt_port_recv_test_ms(t0, t1); + max_ms = nxt_max(max_ms, call_ms); + + if (rc != NXT_UNIT_OK) { + break; + } + } + + end = nxt_unit_test_detached_now(); + + /* + * Each call reports "call me again" until the tenth retry reaches + * the give-up. The give-up closes the worker instead of leaving it + * detached for good. + */ + + if (rc != NXT_UNIT_ERROR || calls <= 10) { + _exit(2); + } + + if (nxt_unit_test_ctx_online(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 11) + { + _exit(3); + } + + if (nxt_port_recv_test_ms(start, end) < 100) { + _exit(4); + } + + /* The last backoff steps are 256 ms. A wait that long is a block. */ + + if (max_ms >= 50) { + _exit(5); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_ms_sleep(int ms) +{ + struct timespec ts; + + ts.tv_sec = ms / 1000; + ts.tv_nsec = (ms % 1000) * 1000000L; + + (void) nanosleep(&ts, NULL); +} + + +static void +nxt_port_recv_test_port_msg_retry_timeout(nxt_unit_ctx_t *ctx, + const char *name) +{ + int rc, timeout; + pid_t pid; + uint64_t end, start; + unsigned int calls; + + /* + * An embedder calls again on NXT_UNIT_OK. If it calls at once each + * time, its loop spins through the backoff until the deadline. So + * nxt_unit_detached_retry_timeout() gives the delay to the deadline. + * An embedder that waits for that delay must still reach each retry + * and the give-up, with few calls. The alarm catches a wait for a + * message. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + nxt_unit_test_send_detached_failures(20); + + /* No retry is pending. */ + + if (nxt_unit_detached_retry_timeout(ctx) != -1) { + _exit(1); + } + + nxt_unit_test_ctx_detached_done(ctx); + + /* The first retry is due at once. */ + + if (nxt_unit_test_ctx_detached_retries(ctx) != 1 + || nxt_unit_detached_retry_timeout(ctx) != 0) + { + _exit(2); + } + + alarm(10); + + /* The retry fails, and the call finds no message. */ + + rc = nxt_unit_process_port_msg(ctx, + nxt_unit_test_ctx_read_port(ctx)); + + timeout = nxt_unit_detached_retry_timeout(ctx); + + if (rc != NXT_UNIT_OK || timeout <= 0 || timeout > 256) { + _exit(3); + } + + /* After the deadline, the next retry is due at once. */ + + nxt_port_recv_test_ms_sleep(timeout + 1); + + if (nxt_unit_detached_retry_timeout(ctx) != 0) { + _exit(4); + } + + start = nxt_unit_test_detached_now(); + + for (calls = 1; calls <= 10000000; calls++) { + rc = nxt_unit_process_port_msg(ctx, + nxt_unit_test_ctx_read_port(ctx)); + if (rc != NXT_UNIT_OK) { + break; + } + + timeout = nxt_unit_detached_retry_timeout(ctx); + + if (timeout > 0) { + nxt_port_recv_test_ms_sleep(timeout); + } + } + + end = nxt_unit_test_detached_now(); + + if (rc != NXT_UNIT_ERROR || nxt_unit_test_ctx_online(ctx) != 0) { + _exit(5); + } + + /* The give-up leaves nothing pending. */ + + if (nxt_unit_detached_retry_timeout(ctx) != -1) { + _exit(6); + } + + /* The embedder waited: few calls, and the backoff still paced. */ + + if (calls > 30 || nxt_port_recv_test_ms(start, end) < 500) { + _exit(7); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + +static void +nxt_port_recv_test_shared_quit_retry(nxt_unit_ctx_t *ctx, unsigned int fails, + int expect, const char *name) +{ + int rc; + pid_t pid; + uint64_t end, start; + + /* + * The shared loop ends when nxt_unit_chk_ready() turns false. That + * can happen with a finish retry pending: a graceful quit deferred on + * the detached state clears ->ready, and the request that ended the + * detached work can be the one that reached "request_limit". The loop + * must run the retry before it leaves. If not, the router holds the + * worker detached, and the deferred quit never completes. Nothing is + * ready, so no read reaches the shared port. The alarm catches a wait + * anyway. + */ + + pid = fork(); + + if (pid == 0) { + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 1); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + nxt_unit_test_ctx_quit_graceful(ctx); + + if (nxt_unit_test_ctx_ready(ctx) != 0 + || nxt_unit_test_ctx_online(ctx) != 1) + { + _exit(1); + } + + nxt_unit_test_send_detached_failures(fails); + nxt_unit_test_ctx_detached_done(ctx); + + if (nxt_unit_test_ctx_detached_retries(ctx) != 1) { + _exit(2); + } + + alarm(5); + + start = nxt_unit_test_detached_now(); + + rc = nxt_unit_run_shared(ctx); + + end = nxt_unit_test_detached_now(); + + if (rc != expect || nxt_unit_test_ctx_online(ctx) != 0) { + _exit(3); + } + + /* + * A retry that succeeds clears the state and completes the quit. + * A retry that never succeeds reaches the give-up. The give-up + * closes the worker instead of leaving it detached. + */ + + if (expect == NXT_UNIT_OK) { + if (nxt_unit_test_ctx_detached(ctx) != 0 + || nxt_unit_test_ctx_detached_retries(ctx) != 0) + { + _exit(4); + } + + } else if (nxt_unit_test_ctx_detached_retries(ctx) != 11) { + _exit(5); + + } else if (nxt_port_recv_test_ms(start, end) < 100) { + /* + * The retries use the same backoff as the read loops, about + * 0.77 s in all. A much shorter time means that the loop + * spins and burns the whole budget at once. + */ + + _exit(6); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + static void nxt_port_recv_test_socket_quit(nxt_unit_ctx_t *ctx, uint8_t ready, const char *name) @@ -670,6 +1059,75 @@ nxt_port_recv_test_socket_quit(nxt_unit_ctx_t *ctx, uint8_t ready, } +static void +nxt_port_recv_test_port_msg_quit(nxt_unit_ctx_t *ctx, const char *name) +{ + int rc; + pid_t pid; + ssize_t n; + nxt_unit_port_t *port; + + struct { + nxt_port_msg_t msg; + uint8_t quit_param; + } nxt_packed m; + + /* + * A QUIT that nxt_unit_process_port_msg() processes runs the quit + * callback. The callback of Node.js calls nxt_unit_done(). Then the + * reference of the call is the last one, and the return frees the + * context. A caller that reads ctx after that, for example with + * nxt_unit_detached_retry_timeout(), touches freed memory. So the call + * must return NXT_UNIT_AGAIN, "do not call again", and the caller must + * not use ctx or port after that. The sanitizer catches a use after + * the free. The alarm catches a wait for a message. + */ + + pid = fork(); + + if (pid == 0) { + nxt_port_recv_test_block = 1; + nxt_port_recv_test_quit_done = 1; + nxt_port_recv_test_quit_calls = 0; + + nxt_unit_test_ctx_set_ready(ctx, 1); + nxt_unit_test_ctx_set_detached(ctx, 0); + nxt_unit_test_ctx_set_detached_retries(ctx, 0); + + port = nxt_unit_test_ctx_read_port(ctx); + + memset(&m, 0, sizeof(m)); + + m.msg.pid = getppid(); + m.msg.type = _NXT_PORT_MSG_QUIT; + m.quit_param = NXT_PORT_QUIT_GRACEFUL; + + n = write(nxt_port_recv_test_read_out_fd, &m, sizeof(m)); + if (n != (ssize_t) sizeof(m)) { + _exit(1); + } + + alarm(5); + + rc = nxt_unit_process_port_msg(ctx, port); + + /* ctx and port are freed here. Only rc and the counter are safe. */ + + if (nxt_port_recv_test_quit_calls != 1) { + _exit(2); + } + + if (rc != NXT_UNIT_AGAIN) { + _exit(3); + } + + _exit(0); + } + + nxt_port_recv_test_child_wait(pid, name); +} + + static void nxt_port_recv_test_detached_persistent_fail(nxt_unit_ctx_t *ctx) { @@ -786,6 +1244,7 @@ main(void) init.callbacks.request_handler = nxt_port_recv_test_handler; init.callbacks.port_send = nxt_port_recv_test_send; init.callbacks.port_recv = nxt_port_recv_test_recv; + init.callbacks.quit = nxt_port_recv_test_quit; init.ready_port.id.pid = getpid(); init.ready_port.id.id = 1; @@ -893,6 +1352,18 @@ main(void) "bare socket quit stops a context not ready"); nxt_port_recv_test_socket_quit(ctx, 1, "bare socket quit stops a ready context"); + nxt_port_recv_test_port_msg_quit(ctx, + "nxt_unit_process_port_msg() stops the caller on a quit that frees"); + nxt_port_recv_test_deferred_quit_port_msg(ctx, + "nxt_unit_process_port_msg() runs a pending finish retry"); + nxt_port_recv_test_port_msg_retry_pending(ctx, + "nxt_unit_process_port_msg() asks for a call back, no wait"); + nxt_port_recv_test_port_msg_retry_timeout(ctx, + "nxt_unit_detached_retry_timeout() paces the call back"); + nxt_port_recv_test_shared_quit_retry(ctx, 1, NXT_UNIT_OK, + "nxt_unit_run_shared() runs a pending finish retry"); + nxt_port_recv_test_shared_quit_retry(ctx, 20, NXT_UNIT_ERROR, + "nxt_unit_run_shared() reaches the give-up, not ready"); nxt_port_recv_test_deferred_quit_read(ctx, 0, "nxt_unit_run() returns on a retry that completes a quit"); nxt_port_recv_test_deferred_quit_read(ctx, 1, @@ -900,6 +1371,14 @@ main(void) nxt_port_recv_test_detached_single_fail(ctx); #endif + /* + * Release the library. After main() returns, no root keeps it + * reachable, and LeakSanitizer reports it or not depending on stale + * stack contents. + */ + nxt_unit_done(ctx2); + nxt_unit_done(ctx); + if (nxt_port_recv_test_failures != 0) { printf("port_recv test: %d failure(s)\n", nxt_port_recv_test_failures); From f5f2469da96e413928d85e2a5a2651a675b8130d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 21:59:05 +0200 Subject: [PATCH 324/391] router: put the sender gate of the main port in one table https://github.com/freeunitorg/freeunit/pull/506 (3e7ea214) checks the sender of seven control messages on the router main port. It did this in seven places: three one-line wrappers for the shared handlers (nxt_router_quit_handler, nxt_router_change_file_handler, nxt_router_access_log_handler), one wrapper helper (nxt_router_main_only), and a check at the top of four router handlers (nxt_router_conf_data_handler, nxt_router_app_restart_handler, nxt_router_status_handler, nxt_router_remove_pid_handler). A new gated type needed a new wrapper or a new check. This change puts the gate in one place. nxt_router_gates[] has one row for each gated type: the sender it accepts and the handler it runs. nxt_router_process_port_handlers points each gated type to nxt_router_gate_handler(). That handler calls nxt_router_msg_sender_ok(), which reads the row for the type, compares the sender pid, and refuses the message as before. Then it runs the handler of the row. A type without a row has no sender that passes. To gate one more type, add its row and point its slot to nxt_router_gate_handler(). The later steps of https://github.com/freeunitorg/freeunit/issues/341 need this for NEW_PORT, GET_PORT, GET_MMAP, MMAP and OOSM. The accepted and refused table does not change: - QUIT, CHANGE_FILE and ACCESS_LOG: main. - REMOVE_PID: main, or a registered prototype. - DATA, APP_RESTART and STATUS: the controller. nxt_router_msg_from_proto() now checks the prototype only; the row NXT_ROUTER_SENDER_MAIN_OR_PROTO checks main first. The four handlers lose their inline check. The three wrappers and nxt_router_main_only() are gone. src/nxt_router.c had 201 added lines in 3e7ea214 and has 194 after this change: 114 lines added and 121 removed. The test calls nxt_router_test_msg_sender_ok(), which takes the type from the message, instead of nxt_router_test_msg_sender_is() with a type argument. Its cases do not change. Co-Authored-By: Claude Fable 5.1 --- src/nxt_router.c | 235 +++++++++++++++--------------- src/nxt_router.h | 9 +- src/test/nxt_router_sender_test.c | 4 +- 3 files changed, 119 insertions(+), 129 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 07650a82f..3a039dd09 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -216,19 +216,13 @@ static nxt_int_t nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app); static nxt_bool_t nxt_router_msg_from(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_process_type_t type); -static nxt_bool_t nxt_router_msg_sender_is(nxt_task_t *task, - nxt_port_recv_msg_t *msg, nxt_process_type_t type); -static nxt_bool_t nxt_router_msg_sender_is_proto(nxt_task_t *task, - nxt_port_recv_msg_t *msg); -static void nxt_router_msg_sender_refuse(nxt_task_t *task, +static nxt_bool_t nxt_router_msg_from_proto(nxt_task_t *task, nxt_port_recv_msg_t *msg); -static void nxt_router_main_only(nxt_task_t *task, nxt_port_recv_msg_t *msg, - nxt_port_handler_t handler); -static void nxt_router_quit_handler(nxt_task_t *task, +static nxt_bool_t nxt_router_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg); -static void nxt_router_change_file_handler(nxt_task_t *task, +static void nxt_router_msg_sender_refuse(nxt_task_t *task, nxt_port_recv_msg_t *msg); -static void nxt_router_access_log_handler(nxt_task_t *task, +static void nxt_router_gate_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); @@ -451,18 +445,70 @@ static const nxt_str_t *nxt_app_msg_prefix[] = { }; +/* + * Each application process has the write end of the main port of the + * router, and the sender sets the message type. Thus the router accepts + * a control message only from its normal sender. This table lists the + * gated types. nxt_router_process_port_handlers points each of them to + * nxt_router_gate_handler(), which checks the sender of the row and then + * runs the handler of the row. To gate one more type, add its row here + * and point its slot to nxt_router_gate_handler(). + * + * The main port of the router has no shared memory queue. Thus each + * message comes through the socket, and the kernel gives the pid of the + * sender (SCM_CREDENTIALS). Without SCM_CREDENTIALS, the pid comes from + * the message header, which the sender sets. + * + * The check is only as good as rt->port_by_type[]. NEW_PORT sets that + * slot from the message, and the router does not check its sender yet. + * Thus an application can first send a NEW_PORT with type MAIN and its + * own pid. The NEW_PORT step of #341 closes this. + */ + +typedef enum { + NXT_ROUTER_SENDER_NONE = 0, + NXT_ROUTER_SENDER_MAIN, + NXT_ROUTER_SENDER_CONTROLLER, + NXT_ROUTER_SENDER_MAIN_OR_PROTO, +} nxt_router_sender_t; + + +typedef struct { + nxt_router_sender_t sender; + nxt_port_handler_t handler; +} nxt_router_gate_t; + + +static const nxt_router_gate_t nxt_router_gates[NXT_PORT_MSG_MAX] = { + [_NXT_PORT_MSG_QUIT] = + { NXT_ROUTER_SENDER_MAIN, nxt_signal_quit_handler }, + [_NXT_PORT_MSG_CHANGE_FILE] = + { NXT_ROUTER_SENDER_MAIN, nxt_port_change_log_file_handler }, + [_NXT_PORT_MSG_ACCESS_LOG] = + { NXT_ROUTER_SENDER_MAIN, nxt_router_access_log_reopen_handler }, + [_NXT_PORT_MSG_REMOVE_PID] = + { NXT_ROUTER_SENDER_MAIN_OR_PROTO, nxt_router_remove_pid_handler }, + [_NXT_PORT_MSG_DATA] = + { NXT_ROUTER_SENDER_CONTROLLER, nxt_router_conf_data_handler }, + [_NXT_PORT_MSG_APP_RESTART] = + { NXT_ROUTER_SENDER_CONTROLLER, nxt_router_app_restart_handler }, + [_NXT_PORT_MSG_STATUS] = + { NXT_ROUTER_SENDER_CONTROLLER, nxt_router_status_handler }, +}; + + static const nxt_port_handlers_t nxt_router_process_port_handlers = { - .quit = nxt_router_quit_handler, + .quit = nxt_router_gate_handler, .new_port = nxt_router_new_port_handler, .get_port = nxt_router_get_port_handler, - .change_file = nxt_router_change_file_handler, + .change_file = nxt_router_gate_handler, .mmap = nxt_port_mmap_handler, .get_mmap = nxt_router_get_mmap_handler, - .data = nxt_router_conf_data_handler, - .app_restart = nxt_router_app_restart_handler, - .status = nxt_router_status_handler, - .remove_pid = nxt_router_remove_pid_handler, - .access_log = nxt_router_access_log_handler, + .data = nxt_router_gate_handler, + .app_restart = nxt_router_gate_handler, + .status = nxt_router_gate_handler, + .remove_pid = nxt_router_gate_handler, + .access_log = nxt_router_gate_handler, .rpc_ready = nxt_port_rpc_handler, .rpc_error = nxt_port_rpc_handler, .oosm = nxt_router_oosm_handler, @@ -1550,29 +1596,7 @@ nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } -/* - * Each application process has the write end of the main port of the - * router, and the sender sets the message type. Thus the router accepts - * these control messages only from their normal sender: - * - * - QUIT, CHANGE_FILE and ACCESS_LOG: main. - * - REMOVE_PID: main or a prototype. - * - DATA, APP_RESTART and STATUS: the controller. - * - * The main port of the router has no shared memory queue. Thus each - * message comes through the socket, and the kernel gives the pid of the - * sender (SCM_CREDENTIALS). Without SCM_CREDENTIALS, the pid comes from - * the message header, which the sender sets. - * - * The router reads the ports from rt->port_by_type[] at message time, - * because a restarted router gets them again through NEW_PORT. A NULL - * port refuses the message. - * - * The check is only as good as rt->port_by_type[]. NEW_PORT sets that - * slot from the message, and the router does not check its sender yet. - * Thus an application can first send a NEW_PORT with type MAIN and its - * own pid. The NEW_PORT step of #341 closes this. - */ +/* The sender pid against the port of the process type; see the gate table. */ static nxt_bool_t nxt_router_msg_from(nxt_task_t *task, nxt_port_recv_msg_t *msg, @@ -1580,26 +1604,17 @@ nxt_router_msg_from(nxt_task_t *task, nxt_port_recv_msg_t *msg, { nxt_port_t *port; + /* + * Read at message time, because a restarted router gets the ports + * again through NEW_PORT. A NULL port refuses the message. + */ + port = task->thread->runtime->port_by_type[type]; return (port != NULL && nxt_recv_msg_cmsg_pid(msg) == port->pid); } -static nxt_bool_t -nxt_router_msg_sender_is(nxt_task_t *task, nxt_port_recv_msg_t *msg, - nxt_process_type_t type) -{ - if (nxt_fast_path(nxt_router_msg_from(task, msg, type))) { - return 1; - } - - nxt_router_msg_sender_refuse(task, msg); - - return 0; -} - - /* * A prototype sends REMOVE_PID for a child that it forked * (nxt_proto_child_exited()). The router cannot check that the pid is a @@ -1608,26 +1623,55 @@ nxt_router_msg_sender_is(nxt_task_t *task, nxt_port_recv_msg_t *msg, */ static nxt_bool_t -nxt_router_msg_sender_is_proto(nxt_task_t *task, nxt_port_recv_msg_t *msg) +nxt_router_msg_from_proto(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_bool_t proto; nxt_process_t *process; - if (nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN)) { - return 1; - } - process = nxt_runtime_process_ref(task->thread->runtime, nxt_recv_msg_cmsg_pid(msg)); + if (process == NULL) { + return 0; + } + + proto = (nxt_process_type(process) == NXT_PROCESS_PROTOTYPE); - if (process != NULL) { - proto = (nxt_process_type(process) == NXT_PROCESS_PROTOTYPE); + nxt_process_use(task, process, -1); + + return proto; +} - nxt_process_use(task, process, -1); - if (proto) { - return 1; - } +static nxt_bool_t +nxt_router_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_bool_t ok; + + /* nxt_port_handler() dispatches only a type below NXT_PORT_MSG_MAX. */ + + switch (nxt_router_gates[msg->port_msg.type].sender) { + + case NXT_ROUTER_SENDER_MAIN: + ok = nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN); + break; + + case NXT_ROUTER_SENDER_CONTROLLER: + ok = nxt_router_msg_from(task, msg, NXT_PROCESS_CONTROLLER); + break; + + case NXT_ROUTER_SENDER_MAIN_OR_PROTO: + ok = nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN) + || nxt_router_msg_from_proto(task, msg); + break; + + default: + /* A type without a row has no sender that passes. */ + ok = 0; + break; + } + + if (nxt_fast_path(ok)) { + return 1; } nxt_router_msg_sender_refuse(task, msg); @@ -1656,39 +1700,15 @@ nxt_router_msg_sender_refuse(nxt_task_t *task, nxt_port_recv_msg_t *msg) } -/* Other processes share these handlers, so the router table gets wrappers. */ - static void -nxt_router_main_only(nxt_task_t *task, nxt_port_recv_msg_t *msg, - nxt_port_handler_t handler) +nxt_router_gate_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - if (nxt_fast_path(nxt_router_msg_sender_is(task, msg, NXT_PROCESS_MAIN))) { - handler(task, msg); + if (nxt_fast_path(nxt_router_msg_sender_ok(task, msg))) { + nxt_router_gates[msg->port_msg.type].handler(task, msg); } } -static void -nxt_router_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) -{ - nxt_router_main_only(task, msg, nxt_signal_quit_handler); -} - - -static void -nxt_router_change_file_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) -{ - nxt_router_main_only(task, msg, nxt_port_change_log_file_handler); -} - - -static void -nxt_router_access_log_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) -{ - nxt_router_main_only(task, msg, nxt_router_access_log_reopen_handler); -} - - #if (NXT_TESTS) /* For src/test/nxt_router_sender_test.c. */ @@ -1697,14 +1717,9 @@ nxt_uint_t nxt_router_test_senders_refused; nxt_bool_t -nxt_router_test_msg_sender_is(nxt_task_t *task, nxt_port_recv_msg_t *msg, - nxt_process_type_t type) +nxt_router_test_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - if (type == NXT_PROCESS_PROTOTYPE) { - return nxt_router_msg_sender_is_proto(task, msg); - } - - return nxt_router_msg_sender_is(task, msg, type); + return nxt_router_msg_sender_ok(task, msg); } #endif @@ -1719,12 +1734,6 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_t *port; nxt_router_temp_conf_t *tmcf; - if (nxt_slow_path(!nxt_router_msg_sender_is(task, msg, - NXT_PROCESS_CONTROLLER))) - { - return; - } - port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, msg->port_msg.reply_port); @@ -1822,12 +1831,6 @@ nxt_router_app_restart_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_port_t *proto_port; nxt_port_msg_type_t reply; - if (nxt_slow_path(!nxt_router_msg_sender_is(task, msg, - NXT_PROCESS_CONTROLLER))) - { - return; - } - reply_port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, msg->port_msg.reply_port); @@ -1928,12 +1931,6 @@ nxt_router_status_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_event_engine_t *engine; nxt_status_report_t *report; - if (nxt_slow_path(!nxt_router_msg_sender_is(task, msg, - NXT_PROCESS_CONTROLLER))) - { - return; - } - port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, msg->port_msg.reply_port); @@ -2047,10 +2044,6 @@ nxt_router_remove_pid_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_event_engine_t *engine; - if (nxt_slow_path(!nxt_router_msg_sender_is_proto(task, msg))) { - return; - } - nxt_port_remove_pid_handler(task, msg); nxt_queue_each(engine, &nxt_router->engines, nxt_event_engine_t, link0) diff --git a/src/nxt_router.h b/src/nxt_router.h index 18fd7f2ff..8f475cb72 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -363,12 +363,9 @@ void nxt_router_access_log_reopen_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); #if (NXT_TESTS) -/* - * The sender checks of the router main port, for the router sender test. - * NXT_PROCESS_PROTOTYPE selects the REMOVE_PID check: main or a prototype. - */ -nxt_bool_t nxt_router_test_msg_sender_is(nxt_task_t *task, - nxt_port_recv_msg_t *msg, nxt_process_type_t type); +/* The sender check of the router main port, for the router sender test. */ +nxt_bool_t nxt_router_test_msg_sender_ok(nxt_task_t *task, + nxt_port_recv_msg_t *msg); extern nxt_uint_t nxt_router_test_senders_refused; diff --git a/src/test/nxt_router_sender_test.c b/src/test/nxt_router_sender_test.c index adb447647..0f219deb5 100644 --- a/src/test/nxt_router_sender_test.c +++ b/src/test/nxt_router_sender_test.c @@ -72,7 +72,7 @@ static const nxt_process_type_t nxt_router_sender_test_port_types[] = { typedef struct { const char *name; nxt_uint_t type; - /* The check; NXT_PROCESS_PROTOTYPE means main or a prototype. */ + /* The expected sender; NXT_PROCESS_PROTOTYPE means main or a prototype. */ nxt_process_type_t check; /* The body is safe to run from the correct sender. */ nxt_bool_t run; @@ -228,7 +228,7 @@ nxt_router_sender_test_accepted(nxt_thread_t *thr, nxt_task_t *task, nxt_router_sender_test_msg(&msg, t->type, sender, sender); - res = nxt_router_test_msg_sender_is(task, &msg, t->check); + res = nxt_router_test_msg_sender_ok(task, &msg); NXT_TEST_CHECK(thr->log, res, "router sender test: %s from %s: check " "returned 0", t->name, who); From 55ea9a86c72711a0e98855c99db7b685cd1227e9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 22:44:42 +0200 Subject: [PATCH 325/391] router: check the sender of NEW_PORT, RPC replies and the libunit requests The gate of https://github.com/freeunitorg/freeunit/pull/506 compares the sender pid with the pid of rt->port_by_type[]. NEW_PORT sets that slot, and the router did not check who sent NEW_PORT. Thus an application could send NEW_PORT with type MAIN and its own pid, and then pass every gate. This change gates NEW_PORT and six more types on the router main port, as planned in https://github.com/freeunitorg/freeunit/issues/341. A row of nxt_router_gates[] now has an optional check that runs after the sender check and gives the reason for a refusal. Two sender kinds are new. NOT_APP is main, the controller or a registered prototype. SELF is a registered application process whose kernel pid is the pid in the header. The rows: - NEW_PORT: main, as before. A registered prototype only for an application port: it announces the main port of each worker it starts. An application only for its own application port and with stream 0: libunit announces the port of each context it adds. - RPC_READY and RPC_ERROR: NOT_APP. An application sends its RPC replies to its own ports, not to this one. - GET_PORT: SELF, and the port asked for must be a router port (pid nxt_pid, type ROUTER). libunit asks only for the port of the router engine that sent it a request. - GET_MMAP: SELF, and the reply port must be an application port. This replaces the nxt_assert() in nxt_router_get_mmap_handler(), so a false name no longer aborts a debug router. - MMAP and OOSM: SELF. A refused message gets one alert with the reason, its descriptors are closed, and it gets no reply, as before. nxt_port_new_port_handler() read the body without a size check. A short body got its missing bytes from an earlier message in the same buffer. nxt_port_new_port_msg() now copies the body through nxt_span_copy(), and every process refuses a short NEW_PORT. Limits. nxt_port_rpc_handler() does not compare the sender with the peer of the registration. Thus main, the controller or any prototype can answer any stream of the router. A check per registration needs a change to the shared RPC code and is left for later. The ports of the router engines stay open: their messages can come through the shared memory queue, which carries no pid. An MMAP always comes through the socket, so it could take a SELF check there later. DETACHED keeps its own check, because its test has no process registry. The router sender test now covers the 14 gated types from 6 senders. A wrong sender sends with its own pid in the header and with the pid of the controller or the worker. The test calls the check first, so a failure is a message and not a crash. Ten cases send a correct sender with a wrong body. The port fd test sends a short NEW_PORT. The CHANGES entry of the first gate drops its NEW_PORT limit, and a new entry describes this change. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 22 +- docs/changes.xml | 25 +- src/nxt_port.c | 54 +++- src/nxt_port.h | 2 + src/nxt_router.c | 279 ++++++++++++++--- src/test/nxt_port_fd_test.c | 21 ++ src/test/nxt_router_sender_test.c | 501 ++++++++++++++++++++++++------ 7 files changed, 750 insertions(+), 154 deletions(-) diff --git a/CHANGES b/CHANGES index 19078569d..cef7bc623 100644 --- a/CHANGES +++ b/CHANGES @@ -52,10 +52,24 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 main. REMOVE_PID must come from main or from a prototype. DATA, APP_RESTART and STATUS must come from the controller. The router closes the descriptors of a refused message and does not reply. - This makes the attack harder, but does not stop it yet: the router - reads the pid of main and of the controller from ports that NEW_PORT - registers, and an application can still send a false NEW_PORT - first. A later change checks NEW_PORT. + + *) Security: an application process could send a false NEW_PORT to + the router, register its own pid as main or as the controller, and + then pass the checks above. It could also get the port of main or + of the controller with GET_PORT, attach a shared memory segment to + another process with MMAP, make a debug router abort with GET_MMAP, + and answer a pending request of the router with RPC_READY or + RPC_ERROR, for example with a false certificate. The router now + accepts NEW_PORT from main, from a prototype for an application + port, and from an application for its own application port only. + GET_PORT, GET_MMAP, MMAP and OOSM must come from the application + that the message names, and GET_PORT gives out only ports of the + router. RPC_READY and RPC_ERROR must come from main, the controller + or a prototype. Every process now refuses a NEW_PORT message that is + too short. Limits: the router does not check that an RPC reply comes + from the process that got the request. The ports of the router + threads get messages through a shared memory queue, which gives no + sender pid, so the router does not check senders there. *) Bugfix: when unitd ran as root and the control socket was given to another user or group with --control-user or --control-group, that diff --git a/docs/changes.xml b/docs/changes.xml index f82ec97b2..0a279022e 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -120,10 +120,27 @@ process, redirect the error log, or reopen the access log. The router now checks the sender pid that the kernel gives. QUIT, CHANGE_FILE and ACCESS_LOG must come from main. REMOVE_PID must come from main or from a prototype. DATA, APP_RESTART and STATUS must come from the controller. The router closes -the descriptors of a refused message and does not reply. This makes the -attack harder, but does not stop it yet: the router reads the pid of main and -of the controller from ports that NEW_PORT registers, and an application can -still send a false NEW_PORT first. A later change checks NEW_PORT. +the descriptors of a refused message and does not reply. + + + + + +an application process could send a false NEW_PORT to the router, register +its own pid as main or as the controller, and then pass the checks above. It +could also get the port of main or of the controller with GET_PORT, attach a +shared memory segment to another process with MMAP, make a debug router abort +with GET_MMAP, and answer a pending request of the router with RPC_READY or +RPC_ERROR, for example with a false certificate. The router now accepts +NEW_PORT from main, from a prototype for an application port, and from an +application for its own application port only. GET_PORT, GET_MMAP, MMAP and +OOSM must come from the application that the message names, and GET_PORT gives +out only ports of the router. RPC_READY and RPC_ERROR must come from main, the +controller or a prototype. Every process now refuses a NEW_PORT message that +is too short. Limits: the router does not check that an RPC reply comes from +the process that got the request. The ports of the router threads get +messages through a shared memory queue, which gives no sender pid, so the +router does not check senders there. diff --git a/src/nxt_port.c b/src/nxt_port.c index b06c27020..78e222fef 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -10,6 +10,7 @@ #include #include #include +#include static void nxt_port_remove_pid(nxt_task_t *task, nxt_port_recv_msg_t *msg, @@ -557,12 +558,37 @@ nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, nxt_port_t *new_port, } +/* + * Copies the body of a NEW_PORT message. The sender sets the size. A body + * shorter than nxt_port_msg_new_port_t is an error: the missing bytes would + * come from an earlier message in the same buffer. + */ + +nxt_int_t +nxt_port_new_port_msg(nxt_port_recv_msg_t *msg, nxt_port_msg_new_port_t *out) +{ + nxt_span_t span; + + if (nxt_slow_path(msg->buf == NULL)) { + return NXT_ERROR; + } + + nxt_span_init(&span, msg->buf->mem.pos, msg->buf->mem.free); + + if (nxt_slow_path(nxt_span_copy(&span, out, sizeof(*out)) != 0)) { + return NXT_ERROR; + } + + return NXT_OK; +} + + void nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_port_t *port; nxt_runtime_t *rt; - nxt_port_msg_new_port_t *new_port_msg; + nxt_port_msg_new_port_t new_port_msg; rt = task->thread->runtime; @@ -575,17 +601,21 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) msg->u.new_port = NULL; msg->new_port_created = 0; - new_port_msg = (nxt_port_msg_new_port_t *) msg->buf->mem.pos; + if (nxt_slow_path(nxt_port_new_port_msg(msg, &new_port_msg) != NXT_OK)) { + nxt_alert(task, "process %PI sent a short new port message; refused", + msg->port_msg.pid); - /* TODO check b size and make plain */ + nxt_port_recv_msg_close_fds(msg); + return; + } nxt_debug(task, "new port %d received for process %PI:%d", - msg->fd[0], new_port_msg->pid, new_port_msg->id); + msg->fd[0], new_port_msg.pid, new_port_msg.id); - port = nxt_runtime_port_find(rt, new_port_msg->pid, new_port_msg->id); + port = nxt_runtime_port_find(rt, new_port_msg.pid, new_port_msg.id); if (port != NULL) { - nxt_debug(task, "port %PI:%d already exists", new_port_msg->pid, - new_port_msg->id); + nxt_debug(task, "port %PI:%d already exists", new_port_msg.pid, + new_port_msg.id); msg->u.new_port = port; @@ -622,9 +652,9 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - port = nxt_runtime_process_port_create(task, rt, new_port_msg->pid, - new_port_msg->id, - new_port_msg->type); + port = nxt_runtime_process_port_create(task, rt, new_port_msg.pid, + new_port_msg.id, + new_port_msg.type); if (nxt_slow_path(port == NULL)) { nxt_port_recv_msg_close_fds(msg); return; @@ -638,8 +668,8 @@ nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* The port owns the descriptor now. */ msg->fd[0] = -1; - port->max_size = new_port_msg->max_size; - port->max_share = new_port_msg->max_share; + port->max_size = new_port_msg.max_size; + port->max_share = new_port_msg.max_share; port->socket.task = task; diff --git a/src/nxt_port.h b/src/nxt_port.h index df58937dc..810815dcf 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -758,6 +758,8 @@ void nxt_port_rearm(nxt_task_t *task, nxt_port_t *port); void *nxt_port_queue_mmap(nxt_task_t *task, nxt_fd_t fd, size_t size); void nxt_port_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +nxt_int_t nxt_port_new_port_msg(nxt_port_recv_msg_t *msg, + nxt_port_msg_new_port_t *out); void nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_port_change_log_file_handler(nxt_task_t *task, diff --git a/src/nxt_router.c b/src/nxt_router.c index 3a039dd09..1ad2805cf 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #if (NXT_HAVE_OTEL) @@ -216,11 +217,19 @@ static nxt_int_t nxt_router_start_app_process(nxt_task_t *task, nxt_app_t *app); static nxt_bool_t nxt_router_msg_from(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_process_type_t type); -static nxt_bool_t nxt_router_msg_from_proto(nxt_task_t *task, +static nxt_bool_t nxt_router_pid_is(nxt_task_t *task, nxt_pid_t pid, + nxt_process_type_t type); +static nxt_bool_t nxt_router_msg_from_self(nxt_task_t *task, nxt_port_recv_msg_t *msg); static nxt_bool_t nxt_router_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_router_msg_sender_refuse(nxt_task_t *task, + nxt_port_recv_msg_t *msg, const char *reason); +static const char *nxt_router_new_port_check(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static const char *nxt_router_get_port_check(nxt_task_t *task, + nxt_port_recv_msg_t *msg); +static const char *nxt_router_get_mmap_check(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_router_gate_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); @@ -448,21 +457,24 @@ static const nxt_str_t *nxt_app_msg_prefix[] = { /* * Each application process has the write end of the main port of the * router, and the sender sets the message type. Thus the router accepts - * a control message only from its normal sender. This table lists the - * gated types. nxt_router_process_port_handlers points each of them to - * nxt_router_gate_handler(), which checks the sender of the row and then - * runs the handler of the row. To gate one more type, add its row here - * and point its slot to nxt_router_gate_handler(). + * a message on this port only from its normal sender. This table lists + * the gated types. nxt_router_process_port_handlers points each of them + * to nxt_router_gate_handler(). That handler checks the sender of the + * row, then runs the check of the row if there is one, and then runs the + * handler of the row. To gate one more type, add its row here and point + * its slot to nxt_router_gate_handler(). * * The main port of the router has no shared memory queue. Thus each * message comes through the socket, and the kernel gives the pid of the * sender (SCM_CREDENTIALS). Without SCM_CREDENTIALS, the pid comes from * the message header, which the sender sets. * - * The check is only as good as rt->port_by_type[]. NEW_PORT sets that - * slot from the message, and the router does not check its sender yet. - * Thus an application can first send a NEW_PORT with type MAIN and its - * own pid. The NEW_PORT step of #341 closes this. + * The sender checks read rt->port_by_type[]. NEW_PORT sets that slot, so + * NEW_PORT is gated too: only main can announce a port that is not an + * application port. + * + * The ports of the router engines are not gated. Their messages can come + * through the shared memory queue, which carries no pid. */ typedef enum { @@ -470,48 +482,85 @@ typedef enum { NXT_ROUTER_SENDER_MAIN, NXT_ROUTER_SENDER_CONTROLLER, NXT_ROUTER_SENDER_MAIN_OR_PROTO, + /* Main, the controller or a registered prototype. */ + NXT_ROUTER_SENDER_NOT_APP, + /* A registered application process that names itself in the header. */ + NXT_ROUTER_SENDER_SELF, + /* Any sender. The check of the row decides. */ + NXT_ROUTER_SENDER_ANY, } nxt_router_sender_t; +/* Returns NULL to accept the message, or the reason to refuse it. */ + +typedef const char *(*nxt_router_gate_check_t)(nxt_task_t *task, + nxt_port_recv_msg_t *msg); + + typedef struct { - nxt_router_sender_t sender; - nxt_port_handler_t handler; + nxt_router_sender_t sender; + nxt_router_gate_check_t check; + nxt_port_handler_t handler; } nxt_router_gate_t; static const nxt_router_gate_t nxt_router_gates[NXT_PORT_MSG_MAX] = { [_NXT_PORT_MSG_QUIT] = - { NXT_ROUTER_SENDER_MAIN, nxt_signal_quit_handler }, + { NXT_ROUTER_SENDER_MAIN, NULL, nxt_signal_quit_handler }, [_NXT_PORT_MSG_CHANGE_FILE] = - { NXT_ROUTER_SENDER_MAIN, nxt_port_change_log_file_handler }, + { NXT_ROUTER_SENDER_MAIN, NULL, nxt_port_change_log_file_handler }, [_NXT_PORT_MSG_ACCESS_LOG] = - { NXT_ROUTER_SENDER_MAIN, nxt_router_access_log_reopen_handler }, + { NXT_ROUTER_SENDER_MAIN, NULL, nxt_router_access_log_reopen_handler }, [_NXT_PORT_MSG_REMOVE_PID] = - { NXT_ROUTER_SENDER_MAIN_OR_PROTO, nxt_router_remove_pid_handler }, + { NXT_ROUTER_SENDER_MAIN_OR_PROTO, NULL, + nxt_router_remove_pid_handler }, [_NXT_PORT_MSG_DATA] = - { NXT_ROUTER_SENDER_CONTROLLER, nxt_router_conf_data_handler }, + { NXT_ROUTER_SENDER_CONTROLLER, NULL, nxt_router_conf_data_handler }, [_NXT_PORT_MSG_APP_RESTART] = - { NXT_ROUTER_SENDER_CONTROLLER, nxt_router_app_restart_handler }, + { NXT_ROUTER_SENDER_CONTROLLER, NULL, + nxt_router_app_restart_handler }, [_NXT_PORT_MSG_STATUS] = - { NXT_ROUTER_SENDER_CONTROLLER, nxt_router_status_handler }, + { NXT_ROUTER_SENDER_CONTROLLER, NULL, nxt_router_status_handler }, + [_NXT_PORT_MSG_NEW_PORT] = + { NXT_ROUTER_SENDER_ANY, nxt_router_new_port_check, + nxt_router_new_port_handler }, + [_NXT_PORT_MSG_GET_PORT] = + { NXT_ROUTER_SENDER_SELF, nxt_router_get_port_check, + nxt_router_get_port_handler }, + [_NXT_PORT_MSG_GET_MMAP] = + { NXT_ROUTER_SENDER_SELF, nxt_router_get_mmap_check, + nxt_router_get_mmap_handler }, + [_NXT_PORT_MSG_MMAP] = + { NXT_ROUTER_SENDER_SELF, NULL, nxt_port_mmap_handler }, + [_NXT_PORT_MSG_OOSM] = + { NXT_ROUTER_SENDER_SELF, NULL, nxt_router_oosm_handler }, + /* + * An application sends its RPC replies to its own ports, never here. + * nxt_port_rpc_handler() does not compare the sender with the peer of + * the registration, so any process that passes can answer any stream. + */ + [_NXT_PORT_MSG_RPC_READY] = + { NXT_ROUTER_SENDER_NOT_APP, NULL, nxt_port_rpc_handler }, + [_NXT_PORT_MSG_RPC_ERROR] = + { NXT_ROUTER_SENDER_NOT_APP, NULL, nxt_port_rpc_handler }, }; static const nxt_port_handlers_t nxt_router_process_port_handlers = { .quit = nxt_router_gate_handler, - .new_port = nxt_router_new_port_handler, - .get_port = nxt_router_get_port_handler, + .new_port = nxt_router_gate_handler, + .get_port = nxt_router_gate_handler, .change_file = nxt_router_gate_handler, - .mmap = nxt_port_mmap_handler, - .get_mmap = nxt_router_get_mmap_handler, + .mmap = nxt_router_gate_handler, + .get_mmap = nxt_router_gate_handler, .data = nxt_router_gate_handler, .app_restart = nxt_router_gate_handler, .status = nxt_router_gate_handler, .remove_pid = nxt_router_gate_handler, .access_log = nxt_router_gate_handler, - .rpc_ready = nxt_port_rpc_handler, - .rpc_error = nxt_port_rpc_handler, - .oosm = nxt_router_oosm_handler, + .rpc_ready = nxt_router_gate_handler, + .rpc_error = nxt_router_gate_handler, + .oosm = nxt_router_gate_handler, .detached = nxt_router_detached_handler, }; @@ -1616,40 +1665,61 @@ nxt_router_msg_from(nxt_task_t *task, nxt_port_recv_msg_t *msg, /* - * A prototype sends REMOVE_PID for a child that it forked - * (nxt_proto_child_exited()). The router cannot check that the pid is a - * child of that prototype: a child that stopped during its start has no - * record in the router. Thus any registered prototype passes. + * Whether the pid is a registered process of the type. A prototype sends + * REMOVE_PID for a child that it forked (nxt_proto_child_exited()). The + * router cannot check that the pid is a child of that prototype: a child + * that stopped during its start has no record in the router. Thus any + * registered prototype passes. */ static nxt_bool_t -nxt_router_msg_from_proto(nxt_task_t *task, nxt_port_recv_msg_t *msg) +nxt_router_pid_is(nxt_task_t *task, nxt_pid_t pid, nxt_process_type_t type) { - nxt_bool_t proto; + nxt_bool_t is; nxt_process_t *process; - process = nxt_runtime_process_ref(task->thread->runtime, - nxt_recv_msg_cmsg_pid(msg)); + process = nxt_runtime_process_ref(task->thread->runtime, pid); if (process == NULL) { return 0; } - proto = (nxt_process_type(process) == NXT_PROCESS_PROTOTYPE); + is = (nxt_process_type(process) == type); nxt_process_use(task, process, -1); - return proto; + return is; +} + + +/* + * The handlers of GET_PORT, GET_MMAP, MMAP and OOSM act for the process + * that the header names. Only libunit sends them, for its own process. + */ + +static nxt_bool_t +nxt_router_msg_from_self(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_pid_t pid; + + pid = nxt_recv_msg_cmsg_pid(msg); + + return (pid > 0 && pid == msg->port_msg.pid + && nxt_router_pid_is(task, pid, NXT_PROCESS_APP)); } static nxt_bool_t nxt_router_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - nxt_bool_t ok; + nxt_bool_t ok; + const char *reason; + const nxt_router_gate_t *gate; /* nxt_port_handler() dispatches only a type below NXT_PORT_MSG_MAX. */ - switch (nxt_router_gates[msg->port_msg.type].sender) { + gate = &nxt_router_gates[msg->port_msg.type]; + + switch (gate->sender) { case NXT_ROUTER_SENDER_MAIN: ok = nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN); @@ -1661,7 +1731,23 @@ nxt_router_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg) case NXT_ROUTER_SENDER_MAIN_OR_PROTO: ok = nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN) - || nxt_router_msg_from_proto(task, msg); + || nxt_router_pid_is(task, nxt_recv_msg_cmsg_pid(msg), + NXT_PROCESS_PROTOTYPE); + break; + + case NXT_ROUTER_SENDER_NOT_APP: + ok = nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN) + || nxt_router_msg_from(task, msg, NXT_PROCESS_CONTROLLER) + || nxt_router_pid_is(task, nxt_recv_msg_cmsg_pid(msg), + NXT_PROCESS_PROTOTYPE); + break; + + case NXT_ROUTER_SENDER_SELF: + ok = nxt_router_msg_from_self(task, msg); + break; + + case NXT_ROUTER_SENDER_ANY: + ok = 1; break; default: @@ -1670,11 +1756,17 @@ nxt_router_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg) break; } + reason = "wrong sender"; + if (nxt_fast_path(ok)) { - return 1; + reason = (gate->check != NULL) ? gate->check(task, msg) : NULL; + + if (nxt_fast_path(reason == NULL)) { + return 1; + } } - nxt_router_msg_sender_refuse(task, msg); + nxt_router_msg_sender_refuse(task, msg, reason); return 0; } @@ -1686,11 +1778,12 @@ nxt_router_msg_sender_ok(nxt_task_t *task, nxt_port_recv_msg_t *msg) */ static void -nxt_router_msg_sender_refuse(nxt_task_t *task, nxt_port_recv_msg_t *msg) +nxt_router_msg_sender_refuse(nxt_task_t *task, nxt_port_recv_msg_t *msg, + const char *reason) { nxt_alert(task, "process %PI sent message type %uD claiming process %PI; " - "refused", nxt_recv_msg_cmsg_pid(msg), - (uint32_t) msg->port_msg.type, msg->port_msg.pid); + "refused: %s", nxt_recv_msg_cmsg_pid(msg), + (uint32_t) msg->port_msg.type, msg->port_msg.pid, reason); nxt_port_recv_msg_close_fds(msg); @@ -1700,6 +1793,102 @@ nxt_router_msg_sender_refuse(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Main announces every process. A prototype announces the main port of + * each worker it starts (nxt_port_process_ready_handler()), with the stream + * of the start. libunit announces the port of each context it adds, for + * its own process and with no stream (nxt_unit_send_port()). Thus only + * main can announce a port that is not an application port. + */ + +static const char * +nxt_router_new_port_check(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_msg_new_port_t new_port; + + if (nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN)) { + return NULL; + } + + if (nxt_slow_path(nxt_port_new_port_msg(msg, &new_port) != NXT_OK)) { + return "short new port message"; + } + + if (nxt_slow_path(new_port.type != NXT_PROCESS_APP)) { + return "not an application port"; + } + + if (nxt_router_pid_is(task, nxt_recv_msg_cmsg_pid(msg), + NXT_PROCESS_PROTOTYPE)) + { + return NULL; + } + + if (nxt_slow_path(!nxt_router_msg_from_self(task, msg) + || new_port.pid != msg->port_msg.pid + || msg->port_msg.stream != 0)) + { + return "not a new port of the sender"; + } + + return NULL; +} + + +/* + * libunit asks for the port of a router engine that sent it a request + * (nxt_unit_get_port()). The router does not give out other ports. + */ + +static const char * +nxt_router_get_port_check(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_t *port; + nxt_span_t span; + nxt_port_msg_get_port_t get_port; + + if (nxt_slow_path(msg->buf == NULL)) { + return "short get port message"; + } + + nxt_span_init(&span, msg->buf->mem.pos, msg->buf->mem.free); + + if (nxt_slow_path(nxt_span_copy(&span, &get_port, sizeof(get_port)) != 0)) + { + return "short get port message"; + } + + port = nxt_runtime_port_find(task->thread->runtime, get_port.pid, + get_port.id); + + if (port != NULL + && (port->pid != nxt_pid || port->type != NXT_PROCESS_ROUTER)) + { + return "not a router port"; + } + + return NULL; +} + + +/* The reply goes to a port that the header names. */ + +static const char * +nxt_router_get_mmap_check(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + nxt_port_t *port; + + port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, + msg->port_msg.reply_port); + + if (port != NULL && port->type != NXT_PROCESS_APP) { + return "reply port is not an application port"; + } + + return NULL; +} + + static void nxt_router_gate_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { @@ -8790,7 +8979,7 @@ nxt_router_get_mmap_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) get_mmap_msg = (nxt_port_msg_get_mmap_t *) msg->buf->mem.pos; - nxt_assert(port->type == NXT_PROCESS_APP); + /* nxt_router_get_mmap_check() refused a port of another type. */ if (nxt_slow_path(port->app == NULL)) { nxt_alert(task, "get_mmap_handler: app == NULL for reply port %PI:%d", diff --git a/src/test/nxt_port_fd_test.c b/src/test/nxt_port_fd_test.c index 13f11d817..e5d2cc9ec 100644 --- a/src/test/nxt_port_fd_test.c +++ b/src/test/nxt_port_fd_test.c @@ -474,6 +474,27 @@ nxt_port_fd_test(nxt_thread_t *thr) b->mem.pos = (u_char *) &new_port_msg; msg.buf = b; + /* + * A NEW_PORT body one byte short. The handler refuses it, closes both + * descriptors and creates no port. + */ + b->mem.free = b->mem.pos + sizeof(nxt_port_msg_new_port_t) - 1; + + ret = nxt_port_fd_test_closed(thr, task, &msg, nxt_port_new_port_handler, + 1, "short new port"); + if (nxt_slow_path(ret != NXT_OK)) { + goto done; + } + + if (nxt_slow_path(msg.u.new_port != NULL)) { + nxt_log_alert(thr->log, "port fd test: a short new port message " + "created a port"); + ret = NXT_ERROR; + goto done; + } + + b->mem.free = b->mem.pos + sizeof(nxt_port_msg_new_port_t); + existing = nxt_runtime_process_port_create(task, rt, new_port_msg.pid, new_port_msg.id, new_port_msg.type); diff --git a/src/test/nxt_router_sender_test.c b/src/test/nxt_router_sender_test.c index 0f219deb5..c5a7c8fca 100644 --- a/src/test/nxt_router_sender_test.c +++ b/src/test/nxt_router_sender_test.c @@ -10,24 +10,37 @@ * QUIT, CHANGE_FILE, ACCESS_LOG main * REMOVE_PID main or a registered prototype * DATA, APP_RESTART, STATUS the controller + * RPC_READY, RPC_ERROR main, the controller or a prototype + * GET_PORT, GET_MMAP, MMAP, OOSM a registered worker, for itself + * NEW_PORT main; a prototype for an application + * port; a worker for its own + * application port, with no stream * * The test sends each type through the handler table of the router from * each sender: main, the controller, a prototype, a worker, an unknown pid - * and no credential. A refused message must add one to the refusal - * counter, close its two descriptors, and not run the handler. The fixture - * makes a handler that runs crash: nxt_router is NULL, and the reply port - * that the message names is registered. + * and no credential. A wrong sender sends twice: with its own pid in the + * header, and with the pid of the controller or of the worker. A refused + * message must add one to the refusal counter, close its two descriptors, + * and not run the handler. The fixture makes most handlers that run crash: + * nxt_router is NULL, and the reply ports that the messages name are + * registered. Thus the test first calls the check alone, which reports a + * failure as a message, and then sends the message through the table. * * An accepted message must not change the counter. The test calls the - * check of each type directly. It also sends CHANGE_FILE, DATA, APP_RESTART - * and STATUS through the table: their bodies find no payload or no reply - * port, log that, and return. QUIT, ACCESS_LOG and REMOVE_PID are not sent - * from their sender, because their bodies stop the process or need a - * started router. + * check of each type directly. It also sends CHANGE_FILE, DATA, + * APP_RESTART, STATUS and MMAP through the table: their bodies find no + * payload, no reply port or no segment, log that, and return. The other + * types are not sent from their sender, because their bodies stop the + * process, need a started router, or write to a port. + * + * A table of cases then sends a correct sender with a wrong body: a + * NEW_PORT that is not an application port, not the port of the sender, + * has a stream or is short; a GET_PORT for a port that is not a router + * port; a GET_MMAP that names a reply port that is not an application + * port. * * A second pass runs with no main port and no controller port, as in a - * restarted router before NEW_PORT. Only a prototype passes, with - * REMOVE_PID. + * restarted router before NEW_PORT. */ #include @@ -43,7 +56,10 @@ #if (NXT_USE_CMSG_PID) -/* The senders. The first four have a port; NONE sends no credential. */ +/* + * The senders. The first four have a port; NONE sends no credential. + * ROUTER is not a sender: it owns a port with the pid of this process. + */ enum { NXT_ROUTER_SENDER_TEST_MAIN = 0, @@ -53,7 +69,33 @@ enum { NXT_ROUTER_SENDER_TEST_STRANGER, NXT_ROUTER_SENDER_TEST_NONE, NXT_ROUTER_SENDER_TEST_PORTS = NXT_ROUTER_SENDER_TEST_STRANGER, - NXT_ROUTER_SENDER_TEST_SENDERS = NXT_ROUTER_SENDER_TEST_NONE + 1 + NXT_ROUTER_SENDER_TEST_SENDERS = NXT_ROUTER_SENDER_TEST_NONE + 1, + NXT_ROUTER_SENDER_TEST_ROUTER = NXT_ROUTER_SENDER_TEST_SENDERS, +}; + + +/* The senders that a type accepts. */ + +enum { + NXT_ROUTER_SENDER_TEST_FROM_MAIN = 0, + NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER, + NXT_ROUTER_SENDER_TEST_FROM_MAIN_OR_PROTO, + NXT_ROUTER_SENDER_TEST_FROM_NOT_APP, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_FROM_NEW_PORT, +}; + + +/* The changes to a correct message in the table of cases. */ + +enum { + NXT_ROUTER_SENDER_TEST_PORT_MAIN = 0, + NXT_ROUTER_SENDER_TEST_PORT_CONTROLLER, + NXT_ROUTER_SENDER_TEST_PORT_OF_PROTO, + NXT_ROUTER_SENDER_TEST_STREAM, + NXT_ROUTER_SENDER_TEST_SHORT, + NXT_ROUTER_SENDER_TEST_FOREIGN_PORT, + NXT_ROUTER_SENDER_TEST_REPLY_NOT_APP, }; @@ -69,27 +111,140 @@ static const nxt_process_type_t nxt_router_sender_test_port_types[] = { }; +/* The port ids in the port hash of the fixture. */ + +#define NXT_ROUTER_SENDER_TEST_REPLY_ID 1 +#define NXT_ROUTER_SENDER_TEST_ENGINE_ID 2 +#define NXT_ROUTER_SENDER_TEST_NOT_APP_ID 3 + + +typedef struct { + nxt_uint_t owner; + nxt_port_id_t id; + nxt_process_type_t type; +} nxt_router_sender_test_hashed_t; + + +static const nxt_router_sender_test_hashed_t + nxt_router_sender_test_hashed[] = +{ + /* The reply port that a forged DATA, APP_RESTART or STATUS names. */ + { NXT_ROUTER_SENDER_TEST_CONTROLLER, NXT_ROUTER_SENDER_TEST_REPLY_ID, + NXT_PROCESS_CONTROLLER }, + /* The reply port of the worker for GET_PORT and GET_MMAP. */ + { NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_REPLY_ID, + NXT_PROCESS_APP }, + /* A router engine port: the only kind that GET_PORT gives out. */ + { NXT_ROUTER_SENDER_TEST_ROUTER, NXT_ROUTER_SENDER_TEST_ENGINE_ID, + NXT_PROCESS_ROUTER }, + /* A port of another type with the pid of the worker, for GET_MMAP. */ + { NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_NOT_APP_ID, + NXT_PROCESS_ROUTER }, +}; + + typedef struct { - const char *name; - nxt_uint_t type; - /* The expected sender; NXT_PROCESS_PROTOTYPE means main or a prototype. */ - nxt_process_type_t check; + const char *name; + nxt_uint_t type; + nxt_uint_t from; + /* The pid in the header of a message from a wrong sender. */ + nxt_uint_t claimed; /* The body is safe to run from the correct sender. */ - nxt_bool_t run; + nxt_bool_t run; } nxt_router_sender_test_type_t; static const nxt_router_sender_test_type_t nxt_router_sender_test_types[] = { - { "QUIT", _NXT_PORT_MSG_QUIT, NXT_PROCESS_MAIN, 0 }, - { "CHANGE_FILE", _NXT_PORT_MSG_CHANGE_FILE, NXT_PROCESS_MAIN, 1 }, - { "ACCESS_LOG", _NXT_PORT_MSG_ACCESS_LOG, NXT_PROCESS_MAIN, 0 }, - { "REMOVE_PID", _NXT_PORT_MSG_REMOVE_PID, NXT_PROCESS_PROTOTYPE, 0 }, - { "DATA", _NXT_PORT_MSG_DATA, NXT_PROCESS_CONTROLLER, 1 }, - { "APP_RESTART", _NXT_PORT_MSG_APP_RESTART, NXT_PROCESS_CONTROLLER, 1 }, - { "STATUS", _NXT_PORT_MSG_STATUS, NXT_PROCESS_CONTROLLER, 1 }, + { "QUIT", _NXT_PORT_MSG_QUIT, + NXT_ROUTER_SENDER_TEST_FROM_MAIN, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "CHANGE_FILE", _NXT_PORT_MSG_CHANGE_FILE, + NXT_ROUTER_SENDER_TEST_FROM_MAIN, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 1 }, + { "ACCESS_LOG", _NXT_PORT_MSG_ACCESS_LOG, + NXT_ROUTER_SENDER_TEST_FROM_MAIN, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "REMOVE_PID", _NXT_PORT_MSG_REMOVE_PID, + NXT_ROUTER_SENDER_TEST_FROM_MAIN_OR_PROTO, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "DATA", _NXT_PORT_MSG_DATA, + NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 1 }, + { "APP_RESTART", _NXT_PORT_MSG_APP_RESTART, + NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 1 }, + { "STATUS", _NXT_PORT_MSG_STATUS, + NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 1 }, + { "RPC_READY", _NXT_PORT_MSG_RPC_READY, + NXT_ROUTER_SENDER_TEST_FROM_NOT_APP, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "RPC_ERROR", _NXT_PORT_MSG_RPC_ERROR, + NXT_ROUTER_SENDER_TEST_FROM_NOT_APP, + NXT_ROUTER_SENDER_TEST_CONTROLLER, 0 }, + { "GET_PORT", _NXT_PORT_MSG_GET_PORT, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_WORKER, 0 }, + { "GET_MMAP", _NXT_PORT_MSG_GET_MMAP, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_WORKER, 0 }, + { "MMAP", _NXT_PORT_MSG_MMAP, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_WORKER, 1 }, + { "OOSM", _NXT_PORT_MSG_OOSM, + NXT_ROUTER_SENDER_TEST_FROM_SELF, + NXT_ROUTER_SENDER_TEST_WORKER, 0 }, + { "NEW_PORT", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_FROM_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, 0 }, +}; + + +typedef struct { + const char *name; + nxt_uint_t type; + nxt_uint_t sender; + nxt_uint_t change; +} nxt_router_sender_test_case_t; + + +static const nxt_router_sender_test_case_t nxt_router_sender_test_cases[] = { + { "NEW_PORT of type MAIN", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_PORT_MAIN }, + { "NEW_PORT of type CONTROLLER", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_PROTO, NXT_ROUTER_SENDER_TEST_PORT_CONTROLLER }, + { "NEW_PORT of type MAIN", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_PROTO, NXT_ROUTER_SENDER_TEST_PORT_MAIN }, + { "NEW_PORT for the prototype", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_PORT_OF_PROTO }, + { "NEW_PORT with a stream", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_STREAM }, + { "short NEW_PORT", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_SHORT }, + { "short NEW_PORT", _NXT_PORT_MSG_NEW_PORT, + NXT_ROUTER_SENDER_TEST_PROTO, NXT_ROUTER_SENDER_TEST_SHORT }, + { "GET_PORT for the controller port", _NXT_PORT_MSG_GET_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_FOREIGN_PORT }, + { "short GET_PORT", _NXT_PORT_MSG_GET_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_SHORT }, + { "GET_MMAP to a port that is not an application port", + _NXT_PORT_MSG_GET_MMAP, + NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_REPLY_NOT_APP }, }; +typedef struct { + nxt_port_recv_msg_t msg; + nxt_buf_t buf; + + union { + nxt_port_msg_new_port_t new_port; + nxt_port_msg_get_port_t get_port; + nxt_port_msg_get_mmap_t get_mmap; + } body; +} nxt_router_sender_test_msg_t; + + static nxt_pid_t nxt_router_sender_test_pid(nxt_uint_t sender) { @@ -97,46 +252,149 @@ nxt_router_sender_test_pid(nxt_uint_t sender) return -1; } + if (sender == NXT_ROUTER_SENDER_TEST_ROUTER) { + return nxt_pid; + } + return nxt_pid + 1 + sender; } -/* Whether the check accepts the sender; known: the router has both ports. */ +/* Whether the type accepts the sender; known: the router has both ports. */ static nxt_bool_t -nxt_router_sender_test_accepts(nxt_process_type_t check, nxt_uint_t sender, +nxt_router_sender_test_accepts(nxt_uint_t from, nxt_uint_t sender, nxt_bool_t known) { - switch (check) { + nxt_bool_t main, controller, proto, worker; + + main = (known && sender == NXT_ROUTER_SENDER_TEST_MAIN); + controller = (known && sender == NXT_ROUTER_SENDER_TEST_CONTROLLER); + proto = (sender == NXT_ROUTER_SENDER_TEST_PROTO); + worker = (sender == NXT_ROUTER_SENDER_TEST_WORKER); - case NXT_PROCESS_MAIN: - return (known && sender == NXT_ROUTER_SENDER_TEST_MAIN); + switch (from) { - case NXT_PROCESS_CONTROLLER: - return (known && sender == NXT_ROUTER_SENDER_TEST_CONTROLLER); + case NXT_ROUTER_SENDER_TEST_FROM_MAIN: + return main; + + case NXT_ROUTER_SENDER_TEST_FROM_CONTROLLER: + return controller; + + case NXT_ROUTER_SENDER_TEST_FROM_MAIN_OR_PROTO: + return (main || proto); + + case NXT_ROUTER_SENDER_TEST_FROM_NOT_APP: + return (main || controller || proto); + + case NXT_ROUTER_SENDER_TEST_FROM_SELF: + return worker; default: - return (sender == NXT_ROUTER_SENDER_TEST_PROTO - || (known && sender == NXT_ROUTER_SENDER_TEST_MAIN)); + return (main || proto || worker); } } -/* A message from "sender", with the pid of "claimed" in the header. */ +/* + * A message of the type from "sender", with the pid of "claimed" in the + * header. NEW_PORT announces an application port of "claimed", GET_PORT + * asks for the router engine port, and GET_MMAP asks for segment 0. The + * reply port of each message has the id NXT_ROUTER_SENDER_TEST_REPLY_ID. + */ static void -nxt_router_sender_test_msg(nxt_port_recv_msg_t *msg, nxt_uint_t type, +nxt_router_sender_test_msg(nxt_router_sender_test_msg_t *m, nxt_uint_t type, nxt_uint_t claimed, nxt_uint_t sender) { - nxt_memzero(msg, sizeof(nxt_port_recv_msg_t)); + size_t size; + nxt_port_recv_msg_t *msg; + + nxt_memzero(m, sizeof(nxt_router_sender_test_msg_t)); + + msg = &m->msg; msg->fd[0] = -1; msg->fd[1] = -1; msg->port_msg.pid = nxt_router_sender_test_pid(claimed); - msg->port_msg.reply_port = 1; + msg->port_msg.reply_port = NXT_ROUTER_SENDER_TEST_REPLY_ID; msg->port_msg.type = type; msg->port_msg.last = 1; msg->cmsg_pid = nxt_router_sender_test_pid(sender); + + switch (type) { + + case _NXT_PORT_MSG_NEW_PORT: + m->body.new_port.pid = msg->port_msg.pid; + m->body.new_port.id = 5; + m->body.new_port.type = NXT_PROCESS_APP; + m->body.new_port.max_size = 1024; + m->body.new_port.max_share = 1024; + size = sizeof(nxt_port_msg_new_port_t); + break; + + case _NXT_PORT_MSG_GET_PORT: + m->body.get_port.pid = nxt_pid; + m->body.get_port.id = NXT_ROUTER_SENDER_TEST_ENGINE_ID; + size = sizeof(nxt_port_msg_get_port_t); + break; + + case _NXT_PORT_MSG_GET_MMAP: + m->body.get_mmap.id = 0; + size = sizeof(nxt_port_msg_get_mmap_t); + break; + + default: + /* The other types are sent with no payload. */ + return; + } + + m->buf.mem.start = (u_char *) &m->body; + m->buf.mem.pos = m->buf.mem.start; + m->buf.mem.free = m->buf.mem.start + size; + m->buf.mem.end = m->buf.mem.free; + + msg->buf = &m->buf; +} + + +static void +nxt_router_sender_test_change(nxt_router_sender_test_msg_t *m, + nxt_uint_t change) +{ + switch (change) { + + case NXT_ROUTER_SENDER_TEST_PORT_MAIN: + m->body.new_port.type = NXT_PROCESS_MAIN; + break; + + case NXT_ROUTER_SENDER_TEST_PORT_CONTROLLER: + m->body.new_port.type = NXT_PROCESS_CONTROLLER; + break; + + case NXT_ROUTER_SENDER_TEST_PORT_OF_PROTO: + m->body.new_port.pid = + nxt_router_sender_test_pid(NXT_ROUTER_SENDER_TEST_PROTO); + break; + + case NXT_ROUTER_SENDER_TEST_STREAM: + m->msg.port_msg.stream = 7; + break; + + case NXT_ROUTER_SENDER_TEST_SHORT: + m->buf.mem.free--; + break; + + case NXT_ROUTER_SENDER_TEST_FOREIGN_PORT: + m->body.get_port.pid = + nxt_router_sender_test_pid(NXT_ROUTER_SENDER_TEST_CONTROLLER); + m->body.get_port.id = NXT_ROUTER_SENDER_TEST_REPLY_ID; + break; + + default: + m->msg.port_msg.reply_port = NXT_ROUTER_SENDER_TEST_NOT_APP_ID; + break; + } } @@ -153,25 +411,35 @@ nxt_router_sender_test_dispatch(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * The check alone must refuse the message. Then the message goes through + * the table with two descriptors, and the router must refuse it again and + * close them. "m" is changed. + */ + static nxt_int_t nxt_router_sender_test_refused(nxt_thread_t *thr, nxt_task_t *task, - const nxt_router_sender_test_type_t *t, nxt_uint_t sender) + nxt_router_sender_test_msg_t *m, const char *name, const char *who) { - nxt_fd_t fd[2]; - nxt_uint_t i, refused; - const char *err; - nxt_port_recv_msg_t msg; - - /* - * The header names the controller, so a DATA, APP_RESTART or STATUS - * body that runs finds the reply port and crashes on nxt_router. - */ - nxt_router_sender_test_msg(&msg, t->type, NXT_ROUTER_SENDER_TEST_CONTROLLER, - sender); + nxt_fd_t fd[2]; + nxt_bool_t res; + nxt_uint_t i, refused; + const char *err; + + refused = nxt_router_test_senders_refused; + + res = nxt_router_test_msg_sender_ok(task, &m->msg); + + NXT_TEST_CHECK(thr->log, !res, "router sender test: %s from %s: check " + "returned 1", name, who); + + NXT_TEST_CHECK(thr->log, nxt_router_test_senders_refused == refused + 1, + "router sender test: %s from %s: check did not count a " + "refusal", name, who); for (i = 0; i < 2; i++) { fd[i] = open("/dev/null", O_RDONLY); - msg.fd[i] = fd[i]; + m->msg.fd[i] = fd[i]; } err = NULL; @@ -183,7 +451,7 @@ nxt_router_sender_test_refused(nxt_thread_t *thr, nxt_task_t *task, refused = nxt_router_test_senders_refused; - nxt_router_sender_test_dispatch(task, &msg); + nxt_router_sender_test_dispatch(task, &m->msg); if (nxt_router_test_senders_refused != refused + 1) { err = "was not refused"; @@ -191,7 +459,7 @@ nxt_router_sender_test_refused(nxt_thread_t *thr, nxt_task_t *task, } else if (nxt_test_fd_is_open(fd[0]) || nxt_test_fd_is_open(fd[1])) { err = "left a descriptor open"; - } else if (msg.fd[0] != -1 || msg.fd[1] != -1) { + } else if (m->msg.fd[0] != -1 || m->msg.fd[1] != -1) { err = "left fds in the message"; } @@ -204,8 +472,8 @@ nxt_router_sender_test_refused(nxt_thread_t *thr, nxt_task_t *task, } if (err != NULL) { - nxt_log_alert(thr->log, "router sender test: %s from %s %s", t->name, - nxt_router_sender_test_names[sender], err); + nxt_log_alert(thr->log, "router sender test: %s from %s %s", name, + who, err); return NXT_ERROR; } @@ -213,32 +481,68 @@ nxt_router_sender_test_refused(nxt_thread_t *thr, nxt_task_t *task, } +/* + * A wrong sender, twice: with its own pid in the header, and with the pid + * of the controller or of the worker. The body of a DATA, APP_RESTART or + * STATUS that names the controller finds the reply port and crashes on + * nxt_router. + */ + +static nxt_int_t +nxt_router_sender_test_wrong(nxt_thread_t *thr, nxt_task_t *task, + const nxt_router_sender_test_type_t *t, nxt_uint_t sender) +{ + nxt_int_t ret; + const char *who; + nxt_router_sender_test_msg_t m; + + who = nxt_router_sender_test_names[sender]; + + nxt_router_sender_test_msg(&m, t->type, sender, sender); + + ret = nxt_router_sender_test_refused(thr, task, &m, t->name, who); + + if (ret == NXT_OK && sender != t->claimed) { + nxt_router_sender_test_msg(&m, t->type, t->claimed, sender); + + ret = nxt_router_sender_test_refused(thr, task, &m, t->name, who); + } + + return ret; +} + + static nxt_int_t nxt_router_sender_test_accepted(nxt_thread_t *thr, nxt_task_t *task, const nxt_router_sender_test_type_t *t, nxt_uint_t sender) { - nxt_bool_t res; - nxt_uint_t refused; - const char *who; - nxt_port_recv_msg_t msg; + nxt_bool_t res; + nxt_uint_t refused, claimed; + const char *who; + nxt_router_sender_test_msg_t m; who = nxt_router_sender_test_names[sender]; refused = nxt_router_test_senders_refused; - nxt_router_sender_test_msg(&msg, t->type, sender, sender); + nxt_router_sender_test_msg(&m, t->type, sender, sender); - res = nxt_router_test_msg_sender_ok(task, &msg); + res = nxt_router_test_msg_sender_ok(task, &m.msg); NXT_TEST_CHECK(thr->log, res, "router sender test: %s from %s: check " "returned 0", t->name, who); if (t->run) { - /* The header names a stranger: the body finds no reply port. */ - nxt_router_sender_test_msg(&msg, t->type, - NXT_ROUTER_SENDER_TEST_STRANGER, sender); + /* + * The header names a stranger: the body finds no reply port. A + * worker must name itself. + */ + claimed = (t->from == NXT_ROUTER_SENDER_TEST_FROM_SELF) + ? sender : NXT_ROUTER_SENDER_TEST_STRANGER; - nxt_router_sender_test_dispatch(task, &msg); + nxt_router_sender_test_msg(&m, t->type, claimed, sender); + + nxt_router_sender_test_dispatch(task, &m.msg); } NXT_TEST_CHECK(thr->log, nxt_router_test_senders_refused == refused, @@ -253,20 +557,22 @@ static nxt_int_t nxt_router_sender_test_run(nxt_thread_t *thr, nxt_task_t *task, nxt_bool_t known) { - nxt_int_t ret; - nxt_uint_t i, s; + nxt_int_t ret; + nxt_uint_t i, s; + nxt_router_sender_test_msg_t m; + const nxt_router_sender_test_case_t *c; const nxt_router_sender_test_type_t *t; for (i = 0; i < nxt_nitems(nxt_router_sender_test_types); i++) { t = &nxt_router_sender_test_types[i]; for (s = 0; s < NXT_ROUTER_SENDER_TEST_SENDERS; s++) { - if (nxt_router_sender_test_accepts(t->check, s, known)) { + if (nxt_router_sender_test_accepts(t->from, s, known)) { ret = nxt_router_sender_test_accepted(thr, task, t, s); } else { - ret = nxt_router_sender_test_refused(thr, task, t, s); + ret = nxt_router_sender_test_wrong(thr, task, t, s); } if (ret != NXT_OK) { @@ -275,6 +581,19 @@ nxt_router_sender_test_run(nxt_thread_t *thr, nxt_task_t *task, } } + for (i = 0; i < nxt_nitems(nxt_router_sender_test_cases); i++) { + c = &nxt_router_sender_test_cases[i]; + + nxt_router_sender_test_msg(&m, c->type, c->sender, c->sender); + nxt_router_sender_test_change(&m, c->change); + + ret = nxt_router_sender_test_refused(thr, task, &m, c->name, + nxt_router_sender_test_names[c->sender]); + if (ret != NXT_OK) { + return ret; + } + } + return NXT_OK; } @@ -352,8 +671,9 @@ nxt_router_sender_test(nxt_thread_t *thr) nxt_int_t ret; nxt_uint_t i; nxt_task_t *task; - nxt_port_t *ports[NXT_ROUTER_SENDER_TEST_PORTS], *reply_port; - nxt_bool_t mutex, hashed; + nxt_port_t *port, *ports[NXT_ROUTER_SENDER_TEST_PORTS]; + nxt_port_t *hashed[nxt_nitems(nxt_router_sender_test_hashed)]; + nxt_bool_t mutex; nxt_router_t *saved_router; nxt_runtime_t *rt, *saved_rt; nxt_process_t *processes[NXT_ROUTER_SENDER_TEST_PORTS]; @@ -363,11 +683,10 @@ nxt_router_sender_test(nxt_thread_t *thr) nxt_log_error(NXT_LOG_NOTICE, thr->log, "router sender test started"); ret = NXT_ERROR; - reply_port = NULL; mutex = 0; - hashed = 0; nxt_memzero(ports, sizeof(ports)); + nxt_memzero(hashed, sizeof(hashed)); nxt_memzero(processes, sizeof(processes)); task = thr->task; @@ -424,19 +743,24 @@ nxt_router_sender_test(nxt_thread_t *thr) } } - /* The reply port that a forged DATA, APP_RESTART or STATUS names. */ + /* The ports that nxt_runtime_port_find() finds. */ - reply_port = nxt_router_sender_test_port(task, 1, - NXT_ROUTER_SENDER_TEST_CONTROLLER, - NXT_PROCESS_CONTROLLER); + for (i = 0; i < nxt_nitems(nxt_router_sender_test_hashed); i++) { + port = nxt_router_sender_test_port(task, + nxt_router_sender_test_hashed[i].id, + nxt_router_sender_test_hashed[i].owner, + nxt_router_sender_test_hashed[i].type); + if (nxt_slow_path(port == NULL)) { + goto done; + } - if (nxt_slow_path(reply_port == NULL - || nxt_port_hash_add(&rt->ports, reply_port) != NXT_OK)) - { - goto done; - } + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, port) != NXT_OK)) { + nxt_port_use(task, port, -1); + goto done; + } - hashed = 1; + hashed[i] = port; + } rt->port_by_type[NXT_PROCESS_MAIN] = ports[NXT_ROUTER_SENDER_TEST_MAIN]; rt->port_by_type[NXT_PROCESS_CONTROLLER] = @@ -462,12 +786,11 @@ nxt_router_sender_test(nxt_thread_t *thr) rt->port_by_type[NXT_PROCESS_CONTROLLER] = NULL; } - if (hashed) { - (void) nxt_port_hash_remove(&rt->ports, reply_port); - } - - if (reply_port != NULL) { - nxt_port_use(task, reply_port, -1); + for (i = 0; i < nxt_nitems(nxt_router_sender_test_hashed); i++) { + if (hashed[i] != NULL) { + (void) nxt_port_hash_remove(&rt->ports, hashed[i]); + nxt_port_use(task, hashed[i], -1); + } } for (i = 0; i < NXT_ROUTER_SENDER_TEST_PORTS; i++) { From 24735a41da7d1ae3be17d530611c43e74de89d8b Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 23:56:31 +0200 Subject: [PATCH 326/391] router: refuse GET_PORT for a foreign pid when the port is not in the hash In nxt_router_get_port_check(), asking for a port of another process was checked only if the port was found in the runtime port hash. If an application sent GET_PORT for a foreign pid with an unhashed port id, nxt_runtime_port_find() returned NULL, the check evaluated to false, and the message was accepted through the gate to nxt_router_get_port_handler(), logging "port not found" rather than being refused as "not a router port". A router engine port always has pid == nxt_pid. The check now tests get_port.pid != nxt_pid first, rejecting any foreign pid immediately without an unnecessary runtime port hash lookup. Also check the length of NEW_PORT before checking if the sender is main, so that short payloads from any sender are refused uniformly at the gate, and declare the msg parameter of nxt_port_new_port_msg() as const. The router sender test now tests an unhashed foreign port id (999) in addition to the registered foreign port. Without this change, that case fails with: "router sender test: GET_PORT for an unhashed foreign port from a worker: check returned 1". Co-Authored-By: Gemini 3.8 flash --- src/nxt_port.c | 3 ++- src/nxt_port.h | 2 +- src/nxt_router.c | 16 +++++++++------- src/test/nxt_router_sender_test.c | 10 ++++++++++ 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/nxt_port.c b/src/nxt_port.c index 78e222fef..b88a6fba9 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -565,7 +565,8 @@ nxt_port_send_port(nxt_task_t *task, nxt_port_t *port, nxt_port_t *new_port, */ nxt_int_t -nxt_port_new_port_msg(nxt_port_recv_msg_t *msg, nxt_port_msg_new_port_t *out) +nxt_port_new_port_msg(const nxt_port_recv_msg_t *msg, + nxt_port_msg_new_port_t *out) { nxt_span_t span; diff --git a/src/nxt_port.h b/src/nxt_port.h index 810815dcf..5dc520615 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -758,7 +758,7 @@ void nxt_port_rearm(nxt_task_t *task, nxt_port_t *port); void *nxt_port_queue_mmap(nxt_task_t *task, nxt_fd_t fd, size_t size); void nxt_port_quit_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); -nxt_int_t nxt_port_new_port_msg(nxt_port_recv_msg_t *msg, +nxt_int_t nxt_port_new_port_msg(const nxt_port_recv_msg_t *msg, nxt_port_msg_new_port_t *out); void nxt_port_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/src/nxt_router.c b/src/nxt_router.c index 1ad2805cf..e1da69fd3 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1806,14 +1806,14 @@ nxt_router_new_port_check(nxt_task_t *task, nxt_port_recv_msg_t *msg) { nxt_port_msg_new_port_t new_port; - if (nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN)) { - return NULL; - } - if (nxt_slow_path(nxt_port_new_port_msg(msg, &new_port) != NXT_OK)) { return "short new port message"; } + if (nxt_router_msg_from(task, msg, NXT_PROCESS_MAIN)) { + return NULL; + } + if (nxt_slow_path(new_port.type != NXT_PROCESS_APP)) { return "not an application port"; } @@ -1858,12 +1858,14 @@ nxt_router_get_port_check(nxt_task_t *task, nxt_port_recv_msg_t *msg) return "short get port message"; } + if (get_port.pid != nxt_pid) { + return "not a router port"; + } + port = nxt_runtime_port_find(task->thread->runtime, get_port.pid, get_port.id); - if (port != NULL - && (port->pid != nxt_pid || port->type != NXT_PROCESS_ROUTER)) - { + if (port != NULL && port->type != NXT_PROCESS_ROUTER) { return "not a router port"; } diff --git a/src/test/nxt_router_sender_test.c b/src/test/nxt_router_sender_test.c index c5a7c8fca..90f3f9116 100644 --- a/src/test/nxt_router_sender_test.c +++ b/src/test/nxt_router_sender_test.c @@ -95,6 +95,7 @@ enum { NXT_ROUTER_SENDER_TEST_STREAM, NXT_ROUTER_SENDER_TEST_SHORT, NXT_ROUTER_SENDER_TEST_FOREIGN_PORT, + NXT_ROUTER_SENDER_TEST_FOREIGN_PORT_UNHASHED, NXT_ROUTER_SENDER_TEST_REPLY_NOT_APP, }; @@ -225,6 +226,9 @@ static const nxt_router_sender_test_case_t nxt_router_sender_test_cases[] = { NXT_ROUTER_SENDER_TEST_PROTO, NXT_ROUTER_SENDER_TEST_SHORT }, { "GET_PORT for the controller port", _NXT_PORT_MSG_GET_PORT, NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_FOREIGN_PORT }, + { "GET_PORT for an unhashed foreign port", _NXT_PORT_MSG_GET_PORT, + NXT_ROUTER_SENDER_TEST_WORKER, + NXT_ROUTER_SENDER_TEST_FOREIGN_PORT_UNHASHED }, { "short GET_PORT", _NXT_PORT_MSG_GET_PORT, NXT_ROUTER_SENDER_TEST_WORKER, NXT_ROUTER_SENDER_TEST_SHORT }, { "GET_MMAP to a port that is not an application port", @@ -391,6 +395,12 @@ nxt_router_sender_test_change(nxt_router_sender_test_msg_t *m, m->body.get_port.id = NXT_ROUTER_SENDER_TEST_REPLY_ID; break; + case NXT_ROUTER_SENDER_TEST_FOREIGN_PORT_UNHASHED: + m->body.get_port.pid = + nxt_router_sender_test_pid(NXT_ROUTER_SENDER_TEST_CONTROLLER); + m->body.get_port.id = 999; + break; + default: m->msg.port_msg.reply_port = NXT_ROUTER_SENDER_TEST_NOT_APP_ID; break; From 0fb49d8efa6f2f4243ed1ba31b04412907b77dba Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 23:21:42 +0200 Subject: [PATCH 327/391] ci: run the standalone libunit tests under ASan, LSan and UBSan c-tests-asan ran build/tests only. unit_close_test, unit_port_recv_test and unit_msg_test have their own main() and are not part of it, so no sanitizer saw them. "make tests" already builds all three in that job. c-tests and build-distros run the first two without a sanitizer; unit_msg_test ran in no job. The job now runs all three after build/tests. The new steps set LSAN_OPTIONS=use_stacks=0. LeakSanitizer scans thread stacks conservatively, so a stale pointer on the stack at exit can keep a leaked object reachable. With the two nxt_unit_done() calls removed from its main(), unit_port_recv_test reported the leak in 3 of 10 runs with the default options and in 10 of 10 with use_stacks=0. The build/tests step keeps the default options. unit_msg_test did not call nxt_unit_done(). Its global context pointer kept the library reachable, so LeakSanitizer could not report a leak of anything the library owns. It now releases the library at the end of main(). With LSAN_OPTIONS=use_stacks=0:use_registers=0: use_globals=0:use_tls=0, the remaining reports are 5597 bytes in 7 allocations from libc's tz data and stdio buffer, the same as for the other two programs. Before the change, allocations from nxt_unit_init() and nxt_unit_add_port() were reported too (23669 bytes in 15 allocations). Local results with the job's flags (clang 21, --openssl --zlib added): each program passed 50 of 50 runs with the default options and 50 of 50 with use_stacks=0, with no LeakSanitizer or UBSan report. No sudo is needed: build-distros runs the first two unprivileged, and all three passed unprivileged here. unit_msg_test was built but run by no job. It now runs in c-tests as well as c-tests-asan: the ASan leg is a --debug build and c-tests is not, and NXT_DEBUG changes what runs (nxt_queue_remove() clears the removed link and nxt_assert() is live only in debug). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 40 ++++++++++++++++++++++++++++++++ src/test/nxt_unit_msg_test.c | 7 ++++++ 2 files changed, 47 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b7a39218d..d4f3e4a08 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1018,6 +1018,13 @@ jobs: - name: Run libunit port_recv fd-ownership test run: ./build/unit_port_recv_test + # Also run in c-tests-asan, but that leg is a --debug build and this one + # is not. They execute different code: NXT_DEBUG makes nxt_queue_remove() + # clear the removed link and enables nxt_assert(), so a debug-only run can + # hide a use-after-unlink that the release build would show. + - name: Run libunit malformed port message test + run: ./build/unit_msg_test + # Same C unit-test suite as c-tests above, built with AddressSanitizer and # run with LeakSanitizer's default ASAN_OPTIONS (detect_leaks is on unless # something turns it off). Separate from sanitize.yml: that workflow runs @@ -1061,6 +1068,39 @@ jobs: - name: Run C tests run: sudo ./build/tests + # The standalone libunit test programs are not part of build/tests: each + # has its own main() and links libunit's instrumented archive, and + # "make tests" builds them. c-tests and build-distros run the first two + # without a sanitizer; unit_msg_test runs nowhere else. No sudo: they + # use socketpair()s, fork() and shared memory, and build-distros runs + # them unprivileged. unit_msg_test's children leave with _exit(), which + # skips the leak check, so only its parent process is checked. + # + # LSAN_OPTIONS=use_stacks=0, for these steps only: LeakSanitizer scans + # thread stacks conservatively, so a stale pointer left on the stack at + # exit keeps a leaked object reachable, and whether one is left depends + # on stack layout. With the nxt_unit_done() calls at the end of its + # main() removed, unit_port_recv_test reported the leak in 3 of 10 runs + # with the default options and in 10 of 10 with use_stacks=0. Globals, + # TLS and registers are still roots, so unit_port_recv_test and + # unit_msg_test release the library with nxt_unit_done() before they + # exit; a leak inside libunit is then unreachable and reported every + # time. + - name: Run libunit close-provenance test + env: + LSAN_OPTIONS: use_stacks=0 + run: ./build/unit_close_test + + - name: Run libunit port_recv fd-ownership test + env: + LSAN_OPTIONS: use_stacks=0 + run: ./build/unit_port_recv_test + + - name: Run libunit malformed port message test + env: + LSAN_OPTIONS: use_stacks=0 + run: ./build/unit_msg_test + # OpenSSL 4.0 compile gate. 4.0 removed nothing Unit uses, but it deprecated # more of what Unit still calls, and 5.0 is where the deprecated symbols # actually go away -- so the value here is having a leg that fails when a new diff --git a/src/test/nxt_unit_msg_test.c b/src/test/nxt_unit_msg_test.c index a1566be94..612873018 100644 --- a/src/test/nxt_unit_msg_test.c +++ b/src/test/nxt_unit_msg_test.c @@ -1127,6 +1127,13 @@ main(void) nxt_unit_msg_test_websocket_case, (void *) &short_frame, NXT_UNIT_OK); + /* + * Release the library. Otherwise the global context pointer keeps it + * reachable at exit, and LeakSanitizer does not see a leak of anything + * the library owns. + */ + nxt_unit_done(nxt_unit_msg_test_ctx); + if (nxt_unit_msg_test_failures != 0) { printf("unit msg test: %d failure(s)\n", nxt_unit_msg_test_failures); return 1; From d4c8558fafa11c8ad8fcb1a6ee72a55d8e5e3574 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 23:01:50 +0000 Subject: [PATCH 328/391] cert: follow-ups to certificate replacement Follow-ups to https://github.com/freeunitorg/freeunit/pull/457, from its review and live validation. Name limit. A name longer than the file system allows reached main. The store failed with ENAMETOOLONG, main logged an alert, and the answer was 500. The controller now refuses a name over 255 bytes with 400 "Invalid certificate name.", before it sends anything to main, for PUT and for DELETE (DELETE gave 404 before). 255 is NAME_MAX on Linux, the BSDs, macOS and Solaris. The temporary file of main has the fixed name "certs/.store.tmp", so a name of 255 bytes works. Main checks the length again, because the controller is unprivileged. The OpenAPI bundleName parameter gets minLength 1, maxLength 255 and the pattern "^[^.].*". CERT_GET from the router only. Since 457 the controller sends bundles with CERT_STORE and never reads one back. Main now accepts CERT_GET from the router only, as it accepts CERT_STORE from the controller only. A compromised controller can no longer read a stored bundle back. Fingerprint. "GET /certificates" and "GET /certificates/" get a "fingerprint" member: the SHA-256 digest of the DER form of the first certificate, as "openssl x509 -fingerprint -sha256" prints it. "GET /certificates//fingerprint" gives it alone. A renewal script compares it with the fingerprint of its file and uploads only a new one. Unchanged bundle. A PUT with the same certificates as the stored bundle stores nothing and applies nothing. nxt_cert_info_t keeps the SHA-256 digest of the SHA-256 digests of the certificates, in bundle order. Equal certificates mean an equal key: nxt_cert_bio() checks the key against the first certificate. The answer is the one a store gives, so a deploy hook keyed on "updated." keeps working after a retry. The metadata is marked as not applied when the router refuses the configuration after a store, or when a store fails (main can fail after it renamed the new file, so the file can hold either bundle). Then the next upload of that name is a full store and apply. The mark is only in the memory of the controller: after a restart of the controller, or a change of a file under state/certs/ by hand, an equal bundle can be skipped. docs/acme.md says this. A failed X509_digest() or EVP_Digest() logs an alert. Router leak. Pre-existing. nxt_router_conf_error() destroyed rtcf->mem_pool but not the SSL_CTX objects built for the new socket confs. Each failed reconfiguration leaked the contexts of every TLS listener built before the failure. It now walks creating_sockets and updating_sockets and calls server_free() on each conf with a bundle. Measured with ASan, two TLS listeners, one bundle truncated, two failed configuration attempts: 27547 bytes in 365 allocations before, no report after. server_init() failures. The router got the bundle from nxt_mp_get(), so when SSL_CTX_new() failed, server_free() gave a garbage pointer to SSL_CTX_free(). The router now uses nxt_mp_zget(), and server_init() sets bundle->ctx to NULL before SSL_CTX_new() and after it frees the context. The chain file is closed once on each path: by server_init() when SSL_CTX_new() fails, by nxt_openssl_chain_file() when BIO_new() fails, else by the BIO, and bundle->chain_file becomes -1. Tests: - test_tls.py: a 256-byte name gets 400 and a 255-byte name works; the fingerprint equals the openssl output and changes with a new certificate; the same bundle again keeps the inode and mtime of the file, and a resumed TLS 1.2 ticket session still resumes (skipped when TLS 1.2 is off); a new certificate is stored, compared with the inode of the old file, and applied. - src/test/nxt_openssl_server_init_test.c: NXT_TESTS hooks make SSL_CTX_new() and BIO_new() fail on a bundle filled with 0xa5; the test checks the context, the chain file and a probe descriptor. It makes its own X509_NAME, so it builds with OpenSSL 1.1, 3.x and 4.0. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 13 +- auto/sources | 1 + docs/acme.md | 16 +- docs/changes.xml | 21 +- docs/unit-openapi.yaml | 63 +++- src/nxt_cert.c | 207 ++++++++++++- src/nxt_cert.h | 9 + src/nxt_controller.c | 61 +++- src/nxt_openssl.c | 52 +++- src/nxt_router.c | 55 +++- src/nxt_tls.h | 11 + src/test/nxt_openssl_server_init_test.c | 378 ++++++++++++++++++++++++ src/test/nxt_tests.c | 6 + src/test/nxt_tests.h | 1 + test/test_tls.py | 137 +++++++++ 15 files changed, 989 insertions(+), 42 deletions(-) create mode 100644 src/test/nxt_openssl_server_init_test.c diff --git a/CHANGES b/CHANGES index cef7bc623..738725803 100644 --- a/CHANGES +++ b/CHANGES @@ -133,6 +133,10 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 compiler that does not support a flag only loses that flag. The default is "off", so a build without the option does not change. + *) Feature: "GET /certificates" shows the SHA-256 fingerprint of the + server certificate of every bundle as "fingerprint", in the form + "openssl x509 -fingerprint -sha256" prints it. + *) Feature: "PUT /certificates/" replaces an existing certificate bundle in place. Main writes the bundle to a temporary file and renames it over the old one, so a crash never leaves a partial @@ -141,7 +145,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 handshakes get the new certificate; accepted connections finish with the old one. An external ACME client can renew a certificate with one request and no restart; see docs/acme.md. A bundle over - 1 MiB is refused with 413. The router opens bundles read-only. + 1 MiB is refused with 413, and a name over 255 bytes with 400, for + PUT and for DELETE. A bundle with the same certificates as the + stored one is not stored again and causes no reconfiguration. The + router opens bundles read-only. + + *) Bugfix: the router leaked the TLS contexts it had built for a new + configuration when applying that configuration failed later, for + example in a bundle of a later listener or in an application. *) Bugfix: with "isolation": {"namespaces": {"pid": true}}, a worker that died before it was fully started left its record, port and diff --git a/auto/sources b/auto/sources index e055d3a7d..d530c19ae 100644 --- a/auto/sources +++ b/auto/sources @@ -245,6 +245,7 @@ fi if [ $NXT_OPENSSL = YES ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_OPENSSL_SRCS" + NXT_TEST_SRCS="$NXT_TEST_SRCS src/test/nxt_openssl_server_init_test.c" fi diff --git a/docs/acme.md b/docs/acme.md index ba02a8b65..c8078f83f 100644 --- a/docs/acme.md +++ b/docs/acme.md @@ -24,8 +24,8 @@ This page gives the procedure for operators. - The key must belong to the first certificate. The answer to a bundle with the wrong key is `400 Invalid certificate.`. The old bundle stays in use. -- A name that starts with `.` is reserved. The answer to such a name is - `400 Invalid certificate name.`. +- A name that starts with `.` is reserved, and a name is at most 255 + bytes. The answer to any other name is `400 Invalid certificate name.`. - The answer to a bundle over 1 MiB is `413 Certificate bundle is too large.`. - When the current configuration names the bundle, the answer is @@ -40,8 +40,20 @@ This page gives the procedure for operators. shows the metadata of the new bundle. The next reconfiguration or restart loads the new bundle. Upload a working bundle again, or correct the configuration. +- A bundle with the same certificates as the stored one is not stored + again and causes no reconfiguration. The answer is the one a store + gives. After `500 Certificate stored but not applied.` or + `500 Failed to store certificate.`, the same bundle is stored and + applied again. The controller keeps this state only in memory: after a + restart of the controller, or after a change of a file under + `state/certs/` by hand, the same bundle can be skipped. Then upload a + different bundle first, or restart Unit. - The request waits while another configuration change is in progress, as `PUT /config` does. +- `GET /certificates//fingerprint` gives the SHA-256 fingerprint of + the server certificate, in the form `openssl x509 -fingerprint -sha256` + prints. A script that polls for a renewed file compares the two and + uploads only a new certificate. ## HTTP-01 route on port 80 diff --git a/docs/changes.xml b/docs/changes.xml index 0a279022e..e6e484164 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -202,6 +202,14 @@ without the option does not change. + + +"GET /certificates" shows the SHA-256 fingerprint of the server certificate of +every bundle as "fingerprint", in the form "openssl x509 -fingerprint -sha256" +prints it. + + + "PUT /certificates/<name>" replaces an existing certificate bundle in @@ -211,7 +219,18 @@ configuration names the bundle, the controller applies the configuration again before it answers. New handshakes get the new certificate; accepted connections finish with the old one. An external ACME client can renew a certificate with one request and no restart; see docs/acme.md. A bundle over -1 MiB is refused with 413. The router opens bundles read-only. +1 MiB is refused with 413, and a name over 255 bytes with 400, for PUT and +for DELETE. A bundle with the same certificates as the stored one is not +stored again and causes no reconfiguration. The router opens bundles +read-only. + + + + + +the router leaked the TLS contexts it had built for a new configuration when +applying that configuration failed later, for example in a bundle of a later +listener or in an application. diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index e52016501..fa25cb7e2 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -124,13 +124,16 @@ paths: certificate, accepted connections finish with the old one. Otherwise the answer is `Certificate chain uploaded.`. A bundle whose key does not belong to the first certificate is refused with - 400 `Invalid certificate.`, a name that starts with `.` with 400 - `Invalid certificate name.`, and a bundle over 1 MiB with 413 + 400 `Invalid certificate.`, a name that starts with `.` or is over + 255 bytes with 400 `Invalid certificate name.`, and a bundle over + 1 MiB with 413 `Certificate bundle is too large.`; the old bundle stays in use. 500 `Certificate stored but not applied.` means the bundle is on disk, but the router refused the configuration with it and keeps - the old certificate. The request waits while a reconfiguration is - in progress, as a configuration change does." + the old certificate. A bundle with the same certificates as the + stored one is not stored again and causes no reconfiguration; the + answer is the one a store gives. The request waits while a + reconfiguration is in progress, as a configuration change does." tags: - certificates @@ -193,6 +196,40 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /certificates/{bundleName}/fingerprint: + summary: "Endpoint for the certificate bundle fingerprint" + get: + operationId: getCertBundleFingerprint + summary: "Retrieve the fingerprint of the server certificate" + description: "Retrieves the SHA-256 fingerprint of the first + certificate in a [stored certificate bundle] + (https://unit.nginx.org/certificates/), in the form + `openssl x509 -fingerprint -sha256` prints it." + + tags: + - certificates + + parameters: + - $ref: "#/components/parameters/bundleName" + + responses: + "200": + description: "OK; the certificate bundle exists in the + configuration." + + content: + application/json: + schema: + type: string + + examples: + Fingerprint: + value: "5F:0A:3E:8B:1C:27:9D:44:B6:E1:72:C8:0D:35:9A:F4:\ + 6B:E8:21:7C:D3:90:4E:AB:15:66:F2:8D:3C:71:B9:02" + + "404": + $ref: "#/components/responses/responseNotFound" + /certificates/{bundleName}/chain: summary: "Endpoint for the certificate bundle chain" get: @@ -5649,11 +5686,17 @@ components: bundleName: in: path - description: "A certificate bundle's name" + description: "A certificate bundle's name. It is not empty, it is + 255 bytes or less, and it does not start with \".\" (such names + are reserved for the certificate store). An invalid name gives + 400 on PUT and DELETE." name: bundleName required: true schema: type: string + minLength: 1 + maxLength: 255 + pattern: "^[^.].*" langMod: in: path @@ -5733,6 +5776,8 @@ components: value: bundle: key: "RSA (4096 bits)" + fingerprint: "5F:0A:3E:8B:1C:27:9D:44:B6:E1:72:C8:0D:35:9A:F4:\ + 6B:E8:21:7C:D3:90:4E:AB:15:66:F2:8D:3C:71:B9:02" chain: - subject: common_name: "example.com" @@ -5774,6 +5819,8 @@ components: summary: "Single certificate bundle" value: key: "RSA (4096 bits)" + fingerprint: "5F:0A:3E:8B:1C:27:9D:44:B6:E1:72:C8:0D:35:9A:F4:\ + 6B:E8:21:7C:D3:90:4E:AB:15:66:F2:8D:3C:71:B9:02" chain: - subject: common_name: "example.com" @@ -6692,6 +6739,12 @@ components: type: string description: "Certificate bundle's key type, i. e. RSA, ECDSA, etc." + fingerprint: + type: string + description: "SHA-256 fingerprint of the first certificate in the + bundle, in the form `openssl x509 -fingerprint -sha256` prints + it: uppercase hex, a colon between the bytes." + chain: $ref: "#/components/schemas/certBundleChain" diff --git a/src/nxt_cert.c b/src/nxt_cert.c index cdb9159f3..c42477229 100644 --- a/src/nxt_cert.c +++ b/src/nxt_cert.c @@ -47,10 +47,28 @@ struct nxt_cert_s { }; +/* The size of a SHA-256 digest. */ +#define NXT_CERT_DIGEST_LENGTH 32 + + struct nxt_cert_info_s { nxt_str_t name; nxt_conf_value_t *value; nxt_mp_t *mp; + + /* See nxt_cert_digest(). */ + u_char digest[NXT_CERT_DIGEST_LENGTH]; + + /* + * The disk or the router may not hold this bundle: the router refused + * the configuration with it, or a store failed and the file may hold + * either bundle. See nxt_cert_info_equal(). The flag is only in the + * memory of the controller. A new controller reads the bundles from + * disk with the flag clear, and a file changed by hand is not seen. In + * these cases an equal upload is skipped until a different bundle is + * stored. + */ + uint8_t unapplied; /* 1 bit */ }; @@ -64,6 +82,8 @@ static nxt_cert_t *nxt_cert_fd(nxt_task_t *task, nxt_fd_t fd); static nxt_cert_t *nxt_cert_bio(nxt_task_t *task, BIO *bio); static int nxt_nxt_cert_pem_suffix(char *pem_str, const char *suffix); +static nxt_int_t nxt_cert_digest(nxt_cert_t *cert, u_char *digest); +static nxt_cert_info_t *nxt_cert_info_find(nxt_str_t *name); static nxt_conf_value_t *nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert); static nxt_conf_value_t *nxt_cert_name_details(nxt_mp_t *mp, X509 *x509, nxt_bool_t issuer); @@ -490,8 +510,13 @@ nxt_cert_info_create(nxt_str_t *name, nxt_cert_t *cert) goto fail; } + if (nxt_slow_path(nxt_cert_digest(cert, info->digest) != NXT_OK)) { + goto fail; + } + info->mp = mp; info->value = value; + info->unapplied = 0; return info; @@ -502,6 +527,100 @@ nxt_cert_info_create(nxt_str_t *name, nxt_cert_t *cert) } +/* + * "digest" gets the SHA-256 digest of the SHA-256 digests of the + * certificates, in bundle order. Two bundles with the same digest hold the + * same certificates. They also hold the same key: nxt_cert_bio() has + * checked the key against the first certificate, and one public key has + * one private key. The PEM text may still differ, in line breaks or in + * the key format. The router loads the same contexts from either. + */ +static nxt_int_t +nxt_cert_digest(nxt_cert_t *cert, u_char *digest) +{ + u_char *p, *buf; + nxt_int_t ret; + nxt_uint_t i; + unsigned int len; + + buf = nxt_malloc(cert->count * EVP_MAX_MD_SIZE); + if (nxt_slow_path(buf == NULL)) { + return NXT_ERROR; + } + + ret = NXT_ERROR; + p = buf; + + for (i = 0; i < cert->count; i++) { + if (nxt_slow_path(X509_digest(cert->chain[i], EVP_sha256(), p, &len) + != 1)) + { + nxt_thread_log_alert("X509_digest() failed"); + goto done; + } + + p += len; + } + + if (nxt_fast_path(EVP_Digest(buf, p - buf, digest, &len, EVP_sha256(), + NULL) == 1 + && len == NXT_CERT_DIGEST_LENGTH)) + { + ret = NXT_OK; + + } else { + nxt_thread_log_alert("EVP_Digest() failed"); + } + +done: + + nxt_free(buf); + + return ret; +} + + +/* + * Whether "cert" holds the same certificates as the stored bundle "name", + * and the router has that bundle. The controller then stores nothing and + * applies nothing. After a store whose configuration the router refused, + * this is false until the same name is stored and applied once more, so a + * repeated upload of that bundle is a full one. + */ +nxt_bool_t +nxt_cert_info_equal(nxt_str_t *name, nxt_cert_t *cert) +{ + u_char digest[NXT_CERT_DIGEST_LENGTH]; + nxt_cert_info_t *info; + + info = nxt_cert_info_find(name); + + if (info == NULL || info->unapplied) { + return 0; + } + + if (nxt_slow_path(nxt_cert_digest(cert, digest) != NXT_OK)) { + return 0; + } + + return (memcmp(digest, info->digest, NXT_CERT_DIGEST_LENGTH) == 0); +} + + +/* The controller records whether the router took the stored bundle. */ +void +nxt_cert_info_applied(nxt_str_t *name, nxt_bool_t applied) +{ + nxt_cert_info_t *info; + + info = nxt_cert_info_find(name); + + if (info != NULL) { + info->unapplied = !applied; + } +} + + /* * Put the metadata in the hash. "old" gets the old metadata of the same * name, or NULL when the name is new. The caller keeps "old" until it calls @@ -532,9 +651,13 @@ nxt_cert_info_replace(nxt_cert_info_t *info, nxt_cert_info_t **old) /* - * Undo nxt_cert_info_replace(): put the old metadata back, or remove the - * name when it was new. Both only free memory, so this cannot fail. - * Releases "info". + * Undo nxt_cert_info_replace() after a failed store: put the old metadata + * back, or remove the name when it was new. Both only free memory, so this + * cannot fail. Releases "info". + * + * Main can fail or die after it renamed the new file over the old one, so + * the file can hold either bundle. The old metadata is thus marked as not + * applied, and the next upload of either bundle is a full store. */ void nxt_cert_info_restore(nxt_cert_info_t *info, nxt_cert_info_t *old) @@ -546,6 +669,8 @@ nxt_cert_info_restore(nxt_cert_info_t *info, nxt_cert_info_t *old) lhq.proto = &nxt_cert_info_hash_proto; if (old != NULL) { + old->unapplied = 1; + lhq.replace = 1; lhq.value = old; @@ -568,11 +693,10 @@ nxt_cert_info_release(nxt_cert_info_t *info) } -nxt_conf_value_t * -nxt_cert_info_get(nxt_str_t *name) +static nxt_cert_info_t * +nxt_cert_info_find(nxt_str_t *name) { nxt_int_t ret; - nxt_cert_info_t *info; nxt_lvlhsh_query_t lhq; lhq.key_hash = nxt_djb_hash(name->start, name->length); @@ -584,9 +708,18 @@ nxt_cert_info_get(nxt_str_t *name) return NULL; } - info = lhq.value; + return lhq.value; +} + - return info->value; +nxt_conf_value_t * +nxt_cert_info_get(nxt_str_t *name) +{ + nxt_cert_info_t *info; + + info = nxt_cert_info_find(name); + + return (info != NULL) ? info->value : NULL; } @@ -648,18 +781,22 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) nxt_str_t str; nxt_int_t ret; nxt_uint_t i; + unsigned int md_len; nxt_conf_value_t *object, *chain, *element, *value; - u_char buf[256]; + u_char buf[256], md[EVP_MAX_MD_SIZE]; + + static const u_char hex[] = "0123456789ABCDEF"; static const nxt_str_t key_str = nxt_string("key"); static const nxt_str_t chain_str = nxt_string("chain"); + static const nxt_str_t fingerprint_str = nxt_string("fingerprint"); static const nxt_str_t since_str = nxt_string("since"); static const nxt_str_t until_str = nxt_string("until"); static const nxt_str_t issuer_str = nxt_string("issuer"); static const nxt_str_t subject_str = nxt_string("subject"); static const nxt_str_t validity_str = nxt_string("validity"); - object = nxt_conf_create_object(mp, 2); + object = nxt_conf_create_object(mp, 3); if (nxt_slow_path(object == NULL)) { return NULL; } @@ -702,6 +839,40 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) nxt_conf_set_member_null(object, &key_str, 0); } + /* + * The SHA-256 fingerprint of the server certificate, in the form + * "openssl x509 -fingerprint -sha256" prints: uppercase hex, a colon + * between the bytes. A renewal script compares it with the + * fingerprint of the file it has, and uploads only a new one. + */ + + if (nxt_slow_path(X509_digest(cert->chain[0], EVP_sha256(), md, &md_len) + != 1)) + { + nxt_thread_log_alert("X509_digest() failed"); + return NULL; + } + + end = buf; + + for (i = 0; i < md_len; i++) { + if (i != 0) { + *end++ = ':'; + } + + *end++ = hex[md[i] >> 4]; + *end++ = hex[md[i] & 0x0f]; + } + + str.length = end - buf; + str.start = buf; + + ret = nxt_conf_set_member_string_dup(object, mp, &fingerprint_str, &str, + 1); + if (nxt_slow_path(ret != NXT_OK)) { + return NULL; + } + chain = nxt_conf_create_array(mp, cert->count); if (nxt_slow_path(chain == NULL)) { return NULL; @@ -785,7 +956,7 @@ nxt_cert_details(nxt_mp_t *mp, nxt_cert_t *cert) nxt_conf_set_element(chain, i, element); } - nxt_conf_set_member(object, &chain_str, chain, 1); + nxt_conf_set_member(object, &chain_str, chain, 2); return object; } @@ -1332,8 +1503,13 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* * Look up the sender's port via the kernel-validated PID * (SCM_CREDENTIALS). msg->port_msg.pid is self-declared, so using - * it would let a compromised worker spoof the controller / router - * and pull arbitrary certificate material out of main. + * it would let a compromised worker spoof the router and pull + * arbitrary certificate material out of main. + * + * Only the router reads bundles: it opens every bundle a listener + * names during a reconfiguration. The controller parses a bundle + * from the request body and sends it to main with CERT_STORE; it + * never reads one back. */ port = nxt_runtime_port_find(task->thread->runtime, nxt_recv_msg_cmsg_pid(msg), @@ -1346,9 +1522,7 @@ nxt_cert_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - if (nxt_slow_path(port->type != NXT_PROCESS_CONTROLLER - && port->type != NXT_PROCESS_ROUTER)) - { + if (nxt_slow_path(port->type != NXT_PROCESS_ROUTER)) { nxt_alert(task, "process %PI cannot read certificates", nxt_recv_msg_cmsg_pid(msg)); nxt_port_recv_msg_close_fds(msg); @@ -1546,6 +1720,7 @@ nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_slow_path(p == NULL || (size_t) (p - name.start) + 1 + sizeof(size_t) != used || p == name.start + || (size_t) (p - name.start) > NXT_CERT_NAME_MAX_LENGTH || name.start[0] == '.' || memchr(name.start, '/', p - name.start) != NULL)) { diff --git a/src/nxt_cert.h b/src/nxt_cert.h index 3f3ee9788..3615d5024 100644 --- a/src/nxt_cert.h +++ b/src/nxt_cert.h @@ -20,6 +20,8 @@ nxt_cert_info_t *nxt_cert_info_create(nxt_str_t *name, nxt_cert_t *cert); nxt_int_t nxt_cert_info_replace(nxt_cert_info_t *info, nxt_cert_info_t **old); void nxt_cert_info_restore(nxt_cert_info_t *info, nxt_cert_info_t *old); void nxt_cert_info_release(nxt_cert_info_t *info); +nxt_bool_t nxt_cert_info_equal(nxt_str_t *name, nxt_cert_t *cert); +void nxt_cert_info_applied(nxt_str_t *name, nxt_bool_t applied); nxt_conf_value_t *nxt_cert_info_get(nxt_str_t *name); nxt_conf_value_t *nxt_cert_info_get_all(nxt_mp_t *mp); nxt_int_t nxt_cert_info_delete(nxt_str_t *name); @@ -27,6 +29,13 @@ nxt_int_t nxt_cert_info_delete(nxt_str_t *name); /* A stored bundle is a chain and a key. 1 MiB can hold hundreds. */ #define NXT_CERT_STORE_MAX_SIZE (1024 * 1024) +/* + * A bundle is the file "certs/". 255 is NAME_MAX on Linux, the BSDs, + * macOS and Solaris. The temporary file of main has a fixed name, so the + * store needs no room for a suffix. + */ +#define NXT_CERT_NAME_MAX_LENGTH 255 + nxt_array_t *nxt_cert_store_load(nxt_task_t *task, nxt_mp_t *mem_pool); void nxt_cert_store_release(nxt_array_t *certs); diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 4ef454d05..b28cfeb3b 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -2074,8 +2074,14 @@ nxt_controller_process_cert(nxt_task_t *task, return; } - /* Names starting with "." are reserved for the store's own files. */ - if (name.length == 0 || path != NULL || name.start[0] == '.') { + /* + * Names starting with "." are reserved for the store's own files. A + * name over NAME_MAX cannot be stored, so it is refused here with 400, + * not by main with ENAMETOOLONG and 500. + */ + if (name.length == 0 || path != NULL || name.start[0] == '.' + || name.length > NXT_CERT_NAME_MAX_LENGTH) + { goto invalid_name; } @@ -2099,6 +2105,17 @@ nxt_controller_process_cert(nxt_task_t *task, goto invalid_cert; } + /* + * The same certificates as the stored bundle, and the router has + * them. A store would write the same file, and a reconfiguration + * would build the same contexts. A script that uploads on a timer + * causes no work here. + */ + if (nxt_cert_info_equal(&name, cert)) { + nxt_cert_destroy(cert); + goto unchanged; + } + store = nxt_mp_get(c->mem_pool, sizeof(nxt_controller_cert_store_t)); if (nxt_slow_path(store == NULL)) { nxt_cert_destroy(cert); @@ -2164,6 +2181,25 @@ nxt_controller_process_cert(nxt_task_t *task, nxt_controller_response(task, req, &resp); return; +unchanged: + + /* + * The answer a store gives, so that a script keyed on either text + * keeps working. The postpone check above has found a ready router, + * so a store of a bundle in use would have applied the configuration. + */ + resp.status = 200; + + if (nxt_controller_cert_in_use(&name)) { + resp.title = (u_char *) "Certificate chain updated."; + + } else { + resp.title = (u_char *) "Certificate chain uploaded."; + } + + nxt_controller_response(task, req, &resp); + return; + invalid_name: resp.status = 400; @@ -2277,7 +2313,8 @@ nxt_controller_cert_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, { rc = nxt_controller_conf_send(task, req->conn->mem_pool, nxt_controller_conf.root, - nxt_controller_cert_apply_handler, req); + nxt_controller_cert_apply_handler, + store); if (nxt_fast_path(rc == NXT_OK)) { nxt_queue_insert_head(&nxt_controller_waiting_requests, @@ -2285,6 +2322,8 @@ nxt_controller_cert_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, return; } + nxt_cert_info_applied(&store->name, 0); + resp.status = 500; resp.title = (u_char *) "Certificate stored but not applied."; resp.offset = -1; @@ -2306,16 +2345,24 @@ static void nxt_controller_cert_apply_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_controller_request_t *req; - nxt_controller_response_t resp; + nxt_bool_t applied; + nxt_controller_request_t *req; + nxt_controller_response_t resp; + nxt_controller_cert_store_t *store; - req = data; + store = data; + req = store->req; nxt_queue_remove(&req->link); nxt_memzero(&resp, sizeof(nxt_controller_response_t)); - if (msg->port_msg.type == NXT_PORT_MSG_RPC_READY) { + applied = (msg->port_msg.type == NXT_PORT_MSG_RPC_READY); + + /* Other changes waited, so the name still holds this bundle. */ + nxt_cert_info_applied(&store->name, applied); + + if (applied) { resp.status = 200; resp.title = (u_char *) "Certificate chain updated."; diff --git a/src/nxt_openssl.c b/src/nxt_openssl.c index bf2630d0c..88d4b8ff8 100644 --- a/src/nxt_openssl.c +++ b/src/nxt_openssl.c @@ -161,6 +161,11 @@ const nxt_tls_lib_t nxt_openssl_lib = { }; +#if (NXT_TESTS) +nxt_uint_t nxt_openssl_test_fail; +#endif + + static nxt_conn_io_t nxt_openssl_conn_io = { .read = nxt_conn_io_read, .recvbuf = nxt_openssl_conn_io_recvbuf, @@ -223,16 +228,32 @@ nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp, STACK_OF(X509_NAME) *list; nxt_tls_bundle_conf_t *bundle; + conf = tls_init->conf; + + bundle = conf->bundle; + nxt_assert(bundle != NULL); + + /* nxt_router_conf_error() frees the context of every bundle. */ + bundle->ctx = NULL; + ctx = SSL_CTX_new(SSLv23_server_method()); + +#if (NXT_TESTS) + if (nxt_openssl_test_fail == NXT_OPENSSL_TEST_FAIL_CTX_NEW) { + SSL_CTX_free(ctx); + ctx = NULL; + } +#endif + if (ctx == NULL) { nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_CTX_new() failed"); - return NXT_ERROR; - } - conf = tls_init->conf; + /* nxt_openssl_chain_file() does not run, so close the file here. */ + nxt_fd_close(bundle->chain_file); + bundle->chain_file = -1; - bundle = conf->bundle; - nxt_assert(bundle != NULL); + return NXT_ERROR; + } bundle->ctx = ctx; @@ -360,6 +381,9 @@ nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp, SSL_CTX_free(ctx); + /* nxt_router_conf_error() frees the contexts of the other bundles. */ + bundle->ctx = NULL; + #if (OPENSSL_VERSION_NUMBER >= 0x1010100fL \ && OPENSSL_VERSION_NUMBER < 0x1010101fL) RAND_keep_random_devices_open(0); @@ -383,14 +407,28 @@ nxt_openssl_chain_file(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_conf_t *conf, ret = NXT_ERROR; cert = NULL; + bundle = conf->bundle; + bio = BIO_new(BIO_s_fd()); + +#if (NXT_TESTS) + if (nxt_openssl_test_fail == NXT_OPENSSL_TEST_FAIL_BIO_NEW) { + BIO_free(bio); + bio = NULL; + } +#endif + if (bio == NULL) { + /* No BIO owns the file yet, so close it here. */ + nxt_fd_close(bundle->chain_file); + bundle->chain_file = -1; + goto end; } - bundle = conf->bundle; - + /* The BIO owns the file now: BIO_free() below closes it. */ BIO_set_fd(bio, bundle->chain_file, BIO_CLOSE); + bundle->chain_file = -1; cert = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL); if (cert == NULL) { diff --git a/src/nxt_router.c b/src/nxt_router.c index e1da69fd3..97098ce02 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -247,6 +247,9 @@ static void nxt_router_conf_ready(nxt_task_t *task, nxt_router_temp_conf_t *tmcf); static void nxt_router_conf_send(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_port_msg_type_t type); +#if (NXT_TLS) +static void nxt_router_conf_error_tls(nxt_task_t *task, nxt_queue_t *sockets); +#endif static nxt_int_t nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, u_char *start, u_char *end); @@ -2614,6 +2617,17 @@ nxt_router_conf_error(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) nxt_router_access_log_release(task, &router->lock, rtcf->access_log); +#if (NXT_TLS) + /* + * The new socket confs go with rtcf->mem_pool, but the TLS contexts + * built for them so far are OpenSSL allocations. No connection uses + * them: the listen joints are posted to the engines only on success. + * The confs in keeping_sockets are the old ones and stay in use. + */ + nxt_router_conf_error_tls(task, &creating_sockets); + nxt_router_conf_error_tls(task, &updating_sockets); +#endif + nxt_mp_destroy(rtcf->mem_pool); nxt_router_conf_send(task, tmcf, NXT_PORT_MSG_RPC_ERROR); @@ -2622,6 +2636,34 @@ nxt_router_conf_error(nxt_task_t *task, nxt_router_temp_conf_t *tmcf) } +#if (NXT_TLS) + +static void +nxt_router_conf_error_tls(nxt_task_t *task, nxt_queue_t *sockets) +{ + nxt_queue_link_t *qlk; + nxt_socket_conf_t *skcf; + + for (qlk = nxt_queue_first(sockets); + qlk != nxt_queue_tail(sockets); + qlk = nxt_queue_next(qlk)) + { + skcf = nxt_queue_link_data(qlk, nxt_socket_conf_t, link); + + /* + * A conf without a bundle failed before its first context. A + * bundle whose server_init() failed has a NULL context, and + * SSL_CTX_free(NULL) does nothing. + */ + if (skcf->tls != NULL && skcf->tls->bundle != NULL) { + task->thread->runtime->tls->server_free(task, skcf->tls); + } + } +} + +#endif + + static void nxt_router_conf_send(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_port_msg_type_t type) @@ -4424,7 +4466,12 @@ nxt_router_tls_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, tls->tls_init->conf = tlscf; - bundle = nxt_mp_get(mp, sizeof(nxt_tls_bundle_conf_t)); + /* + * The error path frees bundle->ctx of every bundle in the chain, so + * the bundle must not hold a garbage pointer before server_init() + * sets it. + */ + bundle = nxt_mp_zget(mp, sizeof(nxt_tls_bundle_conf_t)); if (nxt_slow_path(bundle == NULL)) { goto fail; } @@ -4433,9 +4480,11 @@ nxt_router_tls_rpc_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, goto fail; } + /* + * The bundle owns the descriptor now. server_init() closes it on + * every path and sets bundle->chain_file to -1. + */ bundle->chain_file = msg->fd[0]; - - /* The bundle owns the descriptor now. */ msg->fd[0] = -1; bundle->next = tlscf->bundle; diff --git a/src/nxt_tls.h b/src/nxt_tls.h index 0fb6cfce4..fbe878006 100644 --- a/src/nxt_tls.h +++ b/src/nxt_tls.h @@ -98,6 +98,17 @@ extern const nxt_tls_lib_t nxt_openssl_lib; void nxt_cdecl nxt_openssl_log_error(nxt_task_t *task, nxt_uint_t level, const char *fmt, ...); u_char *nxt_openssl_copy_error(u_char *p, u_char *end); + +#if (NXT_TESTS) +/* + * Makes nxt_openssl_server_init() take one of its failure paths, for + * src/test/nxt_openssl_server_init_test.c. + */ +#define NXT_OPENSSL_TEST_FAIL_CTX_NEW 1 +#define NXT_OPENSSL_TEST_FAIL_BIO_NEW 2 + +NXT_EXPORT extern nxt_uint_t nxt_openssl_test_fail; +#endif #endif #endif /* _NXT_TLS_H_INCLUDED_ */ diff --git a/src/test/nxt_openssl_server_init_test.c b/src/test/nxt_openssl_server_init_test.c new file mode 100644 index 000000000..9497fc909 --- /dev/null +++ b/src/test/nxt_openssl_server_init_test.c @@ -0,0 +1,378 @@ +/* + * Copyright (C) FreeUnit Community + */ + +/* + * Regression test for the failure paths of nxt_openssl_server_init() + * (src/nxt_openssl.c). + * + * nxt_router_tls_rpc_handler() links a new bundle into the chain of the + * socket conf and calls server_init(). When that fails, the router calls + * nxt_router_conf_error(), which calls server_free() on the conf, and + * server_free() calls SSL_CTX_free() on the context of every bundle in the + * chain. Two bugs were on this path: + * + * - The bundle came from nxt_mp_get(), and server_init() returned right + * after a failed SSL_CTX_new() without a write to bundle->ctx. So + * server_free() gave a garbage pointer to SSL_CTX_free(). + * + * - The bundle owns the chain file. Only the BIO in + * nxt_openssl_chain_file() closed it, so the descriptor leaked when + * SSL_CTX_new() or BIO_new() failed. + * + * The test calls server_init() and server_free() directly, not through the + * router: a router conf needs an engine, a port and a controller peer, and + * none of them has a part in these two bugs. Each bundle is allocated the + * way the router allocated it before the fix, with nxt_mp_get(), and filled + * with 0xa5, so the test also covers a caller that does not zero it. + * NXT_TESTS hooks in src/nxt_openssl.c make SSL_CTX_new() or BIO_new() + * fail. + * + * After each call the test opens a probe descriptor. It usually gets the + * number of the closed chain file, so a second close of that number from a + * later path closes the probe, and the test sees it. + */ + +#include +#include "nxt_tests.h" + +#include + +#include +#include +#include + + +typedef struct { + nxt_thread_t *thr; + nxt_task_t *task; + nxt_mp_t *mp; + nxt_tls_conf_t *conf; + nxt_tls_init_t tls_init; + const char *path; +} nxt_openssl_si_test_t; + + +static nxt_int_t +nxt_openssl_si_test_write_bundle(nxt_thread_t *thr, const char *path) +{ + int ok; + FILE *f; + X509 *x; + EVP_PKEY *key; + X509_NAME *name; + EVP_PKEY_CTX *kctx; + + ok = 0; + x = NULL; + key = NULL; + name = NULL; + f = NULL; + + kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_ED25519, NULL); + if (kctx == NULL + || EVP_PKEY_keygen_init(kctx) != 1 + || EVP_PKEY_keygen(kctx, &key) != 1) + { + goto done; + } + + x = X509_new(); + name = X509_NAME_new(); + if (x == NULL || name == NULL) { + goto done; + } + + if (X509_set_version(x, 2) != 1 + || ASN1_INTEGER_set(X509_get_serialNumber(x), 1) != 1 + || X509_gmtime_adj(X509_getm_notBefore(x), 0) == NULL + || X509_gmtime_adj(X509_getm_notAfter(x), 3600) == NULL + || X509_set_pubkey(x, key) != 1 + || X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, + (const u_char *) "localhost", -1, -1, 0) + != 1 + || X509_set_subject_name(x, name) != 1 + || X509_set_issuer_name(x, name) != 1 + || X509_sign(x, key, NULL) == 0) + { + goto done; + } + + f = fopen(path, "w"); + if (f == NULL) { + goto done; + } + + ok = PEM_write_X509(f, x) == 1 + && PEM_write_PrivateKey(f, key, NULL, NULL, 0, NULL, NULL) == 1; + +done: + + if (f != NULL && fclose(f) != 0) { + ok = 0; + } + + X509_NAME_free(name); + X509_free(x); + EVP_PKEY_free(key); + EVP_PKEY_CTX_free(kctx); + + if (!ok) { + nxt_log_alert(thr->log, "openssl server init test: cannot write " + "a test bundle to \"%s\"", path); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * Links a new bundle into the chain the way nxt_router_tls_rpc_handler() + * does, and returns its chain file descriptor. + */ + +static nxt_fd_t +nxt_openssl_si_test_add_bundle(nxt_openssl_si_test_t *t) +{ + nxt_fd_t fd; + nxt_tls_bundle_conf_t *bundle; + + fd = open(t->path, O_RDONLY | O_CLOEXEC); + if (fd == -1) { + nxt_log_alert(t->thr->log, "openssl server init test: " + "open(\"%s\") failed %E", t->path, nxt_errno); + return -1; + } + + bundle = nxt_mp_get(t->mp, sizeof(nxt_tls_bundle_conf_t)); + if (bundle == NULL) { + (void) close(fd); + return -1; + } + + /* Garbage, as nxt_mp_get() can give it. */ + nxt_memset(bundle, 0xa5, sizeof(nxt_tls_bundle_conf_t)); + + nxt_str_set(&bundle->name, "test"); + bundle->chain_file = fd; + + bundle->next = t->conf->bundle; + t->conf->bundle = bundle; + + return fd; +} + + +/* + * Runs server_init() on a new bundle and checks the result, the context of + * the bundle and that the chain file is closed. + */ + +static nxt_int_t +nxt_openssl_si_test_init(nxt_openssl_si_test_t *t, const char *name, + nxt_uint_t fail, nxt_bool_t last, nxt_int_t expect) +{ + nxt_fd_t fd; + nxt_int_t ret; + nxt_tls_bundle_conf_t *bundle; + + fd = nxt_openssl_si_test_add_bundle(t); + if (fd == -1) { + return NXT_ERROR; + } + + bundle = t->conf->bundle; + + nxt_openssl_test_fail = fail; + + ret = nxt_openssl_lib.server_init(t->task, t->mp, &t->tls_init, last); + + nxt_openssl_test_fail = 0; + + NXT_TEST_CHECK(t->thr->log, ret == expect, + "openssl server init test: %s: server_init() returned " + "%i, expected %i", name, ret, expect); + + if (nxt_test_fd_is_open(fd)) { + (void) close(fd); + + nxt_log_alert(t->thr->log, "openssl server init test: %s: " + "the chain file %FD is still open", name, fd); + return NXT_ERROR; + } + + NXT_TEST_CHECK(t->thr->log, (bundle->ctx != NULL) == (expect == NXT_OK), + "openssl server init test: %s: bundle->ctx is %p", + name, bundle->ctx); + + /* Nothing may close the number again. */ + NXT_TEST_CHECK(t->thr->log, bundle->chain_file == -1, + "openssl server init test: %s: bundle->chain_file is %FD, " + "expected -1", name, bundle->chain_file); + + return NXT_OK; +} + + +/* + * Opens a probe, calls server_free() as nxt_router_conf_error() does, and + * checks that the probe is still open. + */ + +static nxt_int_t +nxt_openssl_si_test_free(nxt_openssl_si_test_t *t, const char *name) +{ + nxt_fd_t probe; + nxt_bool_t is_open; + + probe = open(t->path, O_RDONLY | O_CLOEXEC); + if (probe == -1) { + nxt_log_alert(t->thr->log, "openssl server init test: " + "open(\"%s\") failed %E", t->path, nxt_errno); + return NXT_ERROR; + } + + nxt_openssl_lib.server_free(t->task, t->conf); + + is_open = nxt_test_fd_is_open(probe); + + if (is_open) { + (void) close(probe); + } + + NXT_TEST_CHECK(t->thr->log, is_open, + "openssl server init test: %s: server_free() closed " + "the probe descriptor %FD", name, probe); + + return NXT_OK; +} + + +static nxt_int_t +nxt_openssl_si_test_case(nxt_openssl_si_test_t *t, const char *name, + nxt_bool_t first_ok, nxt_uint_t fail) +{ + nxt_int_t ret; + + t->conf = nxt_mp_zget(t->mp, sizeof(nxt_tls_conf_t)); + if (t->conf == NULL) { + return NXT_ERROR; + } + + t->conf->no_wait_shutdown = 1; + + nxt_memzero(&t->tls_init, sizeof(nxt_tls_init_t)); + t->tls_init.conf = t->conf; + + if (first_ok) { + /* An earlier certificate of the same socket, with a context. */ + ret = nxt_openssl_si_test_init(t, name, 0, 0, NXT_OK); + if (ret != NXT_OK) { + return ret; + } + } + + ret = nxt_openssl_si_test_init(t, name, fail, 1, + fail == 0 ? NXT_OK : NXT_ERROR); + if (ret != NXT_OK) { + return ret; + } + + return nxt_openssl_si_test_free(t, name); +} + + +nxt_int_t +nxt_openssl_server_init_test(nxt_thread_t *thr) +{ + char path[256]; + const char *tmpdir; + nxt_int_t ret; + nxt_openssl_si_test_t t; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "openssl server init test started"); + + nxt_memzero(&t, sizeof(nxt_openssl_si_test_t)); + + t.thr = thr; + t.task = thr->task; + t.task->thread = thr; + + if (nxt_openssl_lib.library_init(t.task) != NXT_OK) { + return NXT_ERROR; + } + + tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || tmpdir[0] != '/') { + tmpdir = "/tmp"; + } + + (void) nxt_sprintf((u_char *) path, (u_char *) path + sizeof(path), + "%s/nxt_openssl_server_init_test.XXXXXX%Z", tmpdir); + + ret = mkstemp(path); + if (ret == -1) { + nxt_log_alert(thr->log, "openssl server init test: " + "mkstemp(\"%s\") failed %E", path, nxt_errno); + return NXT_ERROR; + } + + (void) close(ret); + + t.path = path; + + ret = NXT_ERROR; + + t.mp = nxt_mp_create(1024, 128, 256, 32); + if (t.mp == NULL) { + goto done; + } + + if (nxt_openssl_si_test_write_bundle(thr, path) != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "SSL_CTX_new() fails", 0, + NXT_OPENSSL_TEST_FAIL_CTX_NEW); + if (ret != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "SSL_CTX_new() fails, second bundle", + 1, NXT_OPENSSL_TEST_FAIL_CTX_NEW); + if (ret != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "BIO_new() fails", 0, + NXT_OPENSSL_TEST_FAIL_BIO_NEW); + if (ret != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "BIO_new() fails, second bundle", 1, + NXT_OPENSSL_TEST_FAIL_BIO_NEW); + if (ret != NXT_OK) { + goto done; + } + + ret = nxt_openssl_si_test_case(&t, "success", 1, 0); + +done: + + (void) unlink(path); + + if (t.mp != NULL) { + nxt_mp_destroy(t.mp); + } + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "openssl server init test passed"); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index f7d9f9f98..9efd89b4d 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -383,6 +383,12 @@ main(int argc, char **argv) return 1; } +#if (NXT_HAVE_OPENSSL) + if (nxt_openssl_server_init_test(thr) != NXT_OK) { + return 1; + } +#endif + #if (NXT_HAVE_CGROUP) if (nxt_cgroup_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 264b00439..c16f912c0 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -109,6 +109,7 @@ nxt_int_t nxt_router_response_parse_test(nxt_thread_t *thr); nxt_int_t nxt_port_frag_test(nxt_thread_t *thr); nxt_int_t nxt_port_release_test(nxt_thread_t *thr); nxt_int_t nxt_nncq_bound_test(nxt_thread_t *thr); +nxt_int_t nxt_openssl_server_init_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_controller_peer_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); diff --git a/test/test_tls.py b/test/test_tls.py index a8dd340cc..0fd49a2ef 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -1,6 +1,7 @@ import io import os import shutil +import socket import ssl import subprocess import time @@ -213,6 +214,97 @@ def test_tls_certificate_update_unused(): ), 'stored' +def test_tls_certificate_update_unchanged(): + client.load('empty') + + client.certificate() + + # Tickets are off by default. With "tickets": true and no key in the + # configuration, the key is a random one of the SSL_CTX that issued the + # ticket. A reconfiguration builds a new context with a new key, so a + # resumed session proves that the listener kept its context. + assert 'success' in client.conf( + { + "pass": "applications/empty", + "tls": {"certificate": "default", "session": {"tickets": True}}, + }, + 'listeners/*:8080', + ) + + path = f'{option.temp_dir}/state/certs/default' + stored = os.stat(path) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + context.maximum_version = ssl.TLSVersion.TLSv1_2 + + def connect(session=None): + with socket.create_connection(('127.0.0.1', 8080)) as sock: + with context.wrap_socket(sock, session=session) as ssock: + ssock.sendall(b'GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + + resp = b'' + while b'\r\n\r\n' not in resp: + resp += ssock.recv(4096) + + assert resp.startswith(b'HTTP/1.1 200'), 'request' + + return ssock.session, ssock.session_reused + + try: + session, reused = connect() + except ssl.SSLError as e: + # Ticket resumption needs TLS 1.2. A system crypto policy can + # turn it off. + pytest.skip(f'TLS 1.2 is not available: {e}') + + assert not reused, 'new session' + + _, reused = connect(session) + assert reused, 'session resumed' + + # The same bundle again. The answer is the one a store gives, but + # main writes nothing, and the router keeps its contexts. + assert ( + client.certificate_load('default').get('success') + == 'Certificate chain updated.' + ), 'unchanged in use' + + stored_default = os.stat(path) + assert (stored.st_ino, stored.st_mtime_ns) == ( + stored_default.st_ino, + stored_default.st_mtime_ns, + ), 'not stored again' + + _, reused = connect(session) + assert reused, 'no reconfiguration' + + # The same bundle under a name no listener uses. + client.certificate('unused') + + path_unused = f'{option.temp_dir}/state/certs/unused' + stored = os.stat(path_unused) + + assert ( + client.certificate_load('unused').get('success') + == 'Certificate chain uploaded.' + ), 'unchanged unused' + + after = os.stat(path_unused) + assert stored.st_ino == after.st_ino, 'unused not stored again' + + # A new certificate is stored and applied, as before. + assert ( + replace_cert().get('success') == 'Certificate chain updated.' + ), 'replaced' + + assert os.stat(path).st_ino != stored_default.st_ino, 'new bundle stored' + + _, reused = connect(session) + assert not reused, 'reconfigured' + + def test_tls_certificate_update_keepalive(): client.load('empty') @@ -355,6 +447,51 @@ def test_tls_certificate_update_long_name(): assert 'chain' in client.conf_get(f'/certificates/{name}'), 'listed' + # One byte more cannot be a file name. The controller refuses it before + # main sees it, so the answer is 400 and not 500. + assert ( + client.conf(bundle, f'/certificates/{name}a').get('error') + == 'Invalid certificate name.' + ), 'name too long' + + assert 'error' in client.conf_get(f'/certificates/{name}a'), 'not stored' + + +def test_tls_certificate_fingerprint(): + client.certificate() + + # The same value that the OpenSSL command line prints, after the "=". + out = subprocess.check_output( + [ + 'openssl', + 'x509', + '-noout', + '-fingerprint', + '-sha256', + '-in', + f'{option.temp_dir}/default.crt', + ] + ) + + expected = out.decode().split('=', 1)[1].strip() + + assert len(expected) == 95, 'openssl fingerprint' + + assert ( + client.conf_get('/certificates/default/fingerprint') == expected + ), 'fingerprint' + + assert ( + client.conf_get('/certificates')['default']['fingerprint'] == expected + ), 'fingerprint listed' + + # A new certificate under the same name gets a new fingerprint. + assert 'success' in replace_cert(), 'replaced' + + assert ( + client.conf_get('/certificates/default/fingerprint') != expected + ), 'fingerprint changed' + def test_tls_certificate_update_sni(): client.load('empty') From 80c24f61832fba74102027a121e00529acc6fcd9 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 01:28:21 +0000 Subject: [PATCH 329/391] fix(conn): accept and idle-close edge cases Three low-severity fixes in the accept path and at shutdown, from the audit of the nxt_conn_t state machine (M-4, M-5, M-11). M-6, the listener whose spare conn or listen event cannot be allocated, is a separate change. M-4: on Linux, when nxt_conn_io_accept() is used (a system without accept4(), or the poll and select engines), a socket that could not be made non-blocking was closed, and then the closed fd was still used. Return after the close, as the fcntl() check above it does. The fd goes into the spare conn only after this check, so on both error returns the spare keeps fd -1, which nxt_conn_free() asserts at a listener release. M-5: when no connection slot was free, the 100 ms retry timer disabled the read event of the listener each time, also when it was already disabled. This gave an epoll_ctl() ENOENT alert every 100 ms until a slot was free. Check nxt_fd_event_is_active() first, as nxt_router.c does. M-11: at shutdown, nxt_runtime_close_idle_connections() closed idle conns but did not free them, and a conn that never got a request has no write_state, so its close would crash. The shutdown pass now closes each idle conn through c->read_state->close_handler, the same contract that the idle close under fd pressure uses. This ends in nxt_h1p_conn_free() or nxt_controller_conn_free(), which also return the sockaddr to the engine cache and release the listener. A conn with no read_state yet (its accept-init work is still queued) is left for the next pass. Tests: nxt_conn_io_accept_test.c (M-4), nxt_conn_close_idle_test.c (M-5) and nxt_runtime_idle_close_test.c (M-11) fail without the fixes. The M-4 test accepts a real TCP connection and fails the non-blocking step with an NXT_TESTS hook (nxt_conn_test_nonblocking_failures). The M-4 test has a fixture engine, listen socket and work queue, so the code before the fix, which went on into nxt_conn_accept(), gives a test failure and not a crash. The M-11 test sets socket.fd to -1 after its own close(), because nxt_conn_free() asserts it. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 11 + auto/sources | 3 + docs/changes.xml | 22 ++ src/nxt_conn.h | 5 + src/nxt_conn_accept.c | 35 +- src/nxt_runtime.c | 81 ++++- src/nxt_runtime.h | 6 + src/test/nxt_conn_close_idle_test.c | 125 +++++++ src/test/nxt_conn_io_accept_test.c | 243 +++++++++++++ src/test/nxt_runtime_idle_close_test.c | 463 +++++++++++++++++++++++++ src/test/nxt_tests.c | 13 + src/test/nxt_tests.h | 3 + 12 files changed, 993 insertions(+), 17 deletions(-) create mode 100644 src/test/nxt_conn_close_idle_test.c create mode 100644 src/test/nxt_conn_io_accept_test.c create mode 100644 src/test/nxt_runtime_idle_close_test.c diff --git a/CHANGES b/CHANGES index 738725803..859348645 100644 --- a/CHANGES +++ b/CHANGES @@ -91,6 +91,17 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 *) Change: a QUIT that fails because the worker already exited is now logged at the info level, not as an alert. + *) Bugfix: when a router thread ran out of connections or file + descriptors, the log got an "epoll_ctl() failed (2: No such file or + directory)" alert every 100 milliseconds until a connection was + freed. + + *) Bugfix: at shutdown, idle client connections were closed but not + freed. They are now closed through their protocol handlers. + + *) Bugfix: on systems without accept4(), an accepted socket that could + not be made non-blocking was closed and then still used. + *) Security: an application could write an unexpected value into an entry of a shared memory queue. The router then retried that entry for ever. Each enqueue and dequeue now stops after one queue size diff --git a/auto/sources b/auto/sources index d530c19ae..1bb4ac920 100644 --- a/auto/sources +++ b/auto/sources @@ -190,6 +190,9 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_mmap_read_test.c \ src/test/nxt_port_mmaps_max_test.c \ src/test/nxt_port_ready_test.c \ + src/test/nxt_conn_close_idle_test.c \ + src/test/nxt_conn_io_accept_test.c \ + src/test/nxt_runtime_idle_close_test.c \ src/test/nxt_router_new_port_test.c \ src/test/nxt_router_start_fail_test.c \ src/test/nxt_router_start_fail_soak_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index e6e484164..81ad22212 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -44,6 +44,28 @@ level, not as an alert. + + +when a router thread ran out of connections or file descriptors, the log got +an "epoll_ctl() failed (2: No such file or directory)" alert every 100 +milliseconds until a connection was freed. + + + + + +at shutdown, idle client connections were closed but not freed. They are now +closed through their protocol handlers. + + + + + +on systems without accept4(), an accepted socket that could not be made +non-blocking was closed and then still used. + + + the router and libunit read records from shared memory that the other diff --git a/src/nxt_conn.h b/src/nxt_conn.h index 4bb51c863..42ee62c28 100644 --- a/src/nxt_conn.h +++ b/src/nxt_conn.h @@ -265,6 +265,11 @@ void nxt_conn_connect_error(nxt_task_t *task, void *obj, void *data); NXT_EXPORT nxt_listen_event_t *nxt_listen_event(nxt_task_t *task, nxt_listen_socket_t *ls); void nxt_conn_io_accept(nxt_task_t *task, void *obj, void *data); + +#if (NXT_TESTS) +extern nxt_uint_t nxt_conn_test_nonblocking_failures; +#endif + NXT_EXPORT void nxt_conn_accept(nxt_task_t *task, nxt_listen_event_t *lev, nxt_conn_t *c); void nxt_conn_accept_error(nxt_task_t *task, nxt_listen_event_t *lev, diff --git a/src/nxt_conn_accept.c b/src/nxt_conn_accept.c index 7e2576705..6efa3ce37 100644 --- a/src/nxt_conn_accept.c +++ b/src/nxt_conn_accept.c @@ -132,9 +132,20 @@ nxt_conn_listen_handler(nxt_task_t *task, void *obj, void *data) } +#if (NXT_TESTS) + +/* For src/test/nxt_conn_io_accept_test.c: fail the next N nonblocking calls. */ +nxt_uint_t nxt_conn_test_nonblocking_failures; + +#endif + + void nxt_conn_io_accept(nxt_task_t *task, void *obj, void *data) { +#if (NXT_LINUX) + nxt_int_t ret; +#endif socklen_t socklen; nxt_conn_t *c; nxt_socket_t s; @@ -177,19 +188,33 @@ nxt_conn_io_accept(nxt_task_t *task, void *obj, void *data) return; } - c->socket.fd = s; - #if (NXT_LINUX) /* * Linux does not inherit non-blocking mode * from listen socket for accept()ed socket. */ - if (nxt_slow_path(nxt_socket_nonblocking(task, s) != NXT_OK)) { + ret = nxt_socket_nonblocking(task, s); + +#if (NXT_TESTS) + if (nxt_conn_test_nonblocking_failures > 0) { + nxt_conn_test_nonblocking_failures--; + ret = NXT_ERROR; + } +#endif + + if (nxt_slow_path(ret != NXT_OK)) { nxt_socket_close(task, s); + return; } #endif + /* + * Only a socket that is kept goes into the spare conn: on the error + * returns above the spare keeps fd -1, which nxt_conn_free() asserts. + */ + c->socket.fd = s; + nxt_debug(task, "accept(%d): %d", lev->socket.fd, s); nxt_conn_accept(task, lev, c); @@ -274,7 +299,9 @@ nxt_conn_accept_close_idle(nxt_task_t *task, nxt_listen_event_t *lev) nxt_timer_add(engine, &lev->timer, 100); - nxt_fd_event_disable_read(engine, &lev->socket); + if (nxt_fd_event_is_active(lev->socket.read)) { + nxt_fd_event_disable_read(engine, &lev->socket); + } nxt_alert(task, "new connections are not accepted within 100ms"); } diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index e65c27b01..91de56231 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -479,6 +479,32 @@ nxt_runtime_quit(nxt_task_t *task, nxt_uint_t status) } +/* + * Close every idle conn of the engine on shutdown. + * + * Each conn is handed to its own protocol through c->read_state's + * close_handler, the same contract nxt_conn_accept_close_idle_handler() + * (nxt_conn_accept.c) uses to reclaim idle conns under fd pressure. That + * handler ends in the protocol's free path -- nxt_h1p_conn_free() for a + * router conn, nxt_controller_conn_free() for a controller conn -- which + * releases everything the runtime cannot know about: the sockaddr from the + * engine cache (nxt_conn_accept_alloc()), and for the router the listener + * reference taken in nxt_conn_accept(), without which a draining listener + * never reaches its final release. It also satisfies CONN-INV-2 (a + * write_state whose ready_handler releases the conn is installed before + * nxt_conn_close()), which a bare nxt_conn_close() here would violate: a + * keep-alive h1p conn still carries its request-send state, whose + * ready_handler is a no-op with c->write == NULL, so the conn would leak. + * + * A conn with no read_state has been accepted but its listen handler + * (nxt_conn_accept(): lev->listen->handler, queued on the read work queue) + * has not run yet, so no protocol owns it. It is left alone: that queued + * work item still references it, so closing or freeing it here would be a + * use-after-free once the item runs, and nxt_conn_close() on a conn with + * write_state == NULL crashes in nxt_conn_close_handler(). It gets its + * protocol state when that item runs and is closed by the next pass. + */ + static void nxt_runtime_close_idle_connections(nxt_event_engine_t *engine) { @@ -497,24 +523,53 @@ nxt_runtime_close_idle_connections(nxt_event_engine_t *engine) next = nxt_queue_next(link); c = nxt_queue_link_data(link, nxt_conn_t, link); - if (!c->socket.read_ready) { - /* - * Unlink and clear the tracking state immediately, before - * scheduling the async close. nxt_runtime_quit() calls this on - * every shutdown continuation, so a conn left on idle_connections - * would be re-selected and re-closed on the next pass before its - * async close handler runs -- a double nxt_conn_close(), i.e. a - * use-after-free. Clearing c->idle to TRACK_NONE also stops the - * close handler from unlinking the same conn a second time - * (P4.5). Iteration stays safe: `next` was captured above. - */ - nxt_conn_untrack(engine, c); - nxt_conn_close(engine, c); + if (c->socket.read_ready) { + continue; } + + if (c->read_state == NULL || c->read_state->close_handler == NULL) { + nxt_debug(c->socket.task, "idle connection: %d not yet " + "initialized, skipped", c->socket.fd); + continue; + } + + /* + * Unlink and clear the tracking state immediately, before the + * protocol schedules its async close. nxt_runtime_quit() calls + * this on every shutdown continuation, so a conn left on + * idle_connections would be re-selected and re-closed on the next + * pass before its async close handler runs -- a double + * nxt_conn_close(), i.e. a use-after-free. Clearing c->idle to + * TRACK_NONE also stops the close handler from unlinking the same + * conn a second time (P4.5), and makes the nxt_conn_active() at + * the top of the h1p close handlers a no-op. Iteration stays + * safe: `next` was captured above. + */ + nxt_conn_untrack(engine, c); + + c->read_state->close_handler(c->socket.task, c, c->socket.data); } } +#if (NXT_TESTS) + +/* + * src/test/nxt_runtime_idle_close_test.c drives the idle-connection close + * path (M-11) without going through a full nxt_runtime_quit(): it checks + * that every idle conn is closed through its own protocol close handler, + * and that a conn without one is left alone. + */ + +void +nxt_runtime_test_close_idle_connections(nxt_event_engine_t *engine) +{ + nxt_runtime_close_idle_connections(engine); +} + +#endif + + /* * Allocate a one-byte port-message body carrying rt->quit_mode (a * nxt_port_quit_mode_t value, see nxt_port.h). libunit parses this diff --git a/src/nxt_runtime.h b/src/nxt_runtime.h index 15c2ec8e4..e24624382 100644 --- a/src/nxt_runtime.h +++ b/src/nxt_runtime.h @@ -189,4 +189,10 @@ extern nxt_module_init_t nxt_init_modules[]; extern nxt_uint_t nxt_init_modules_n; +#if (NXT_TESTS) +/* The idle-connection close pass, for the idle close dispatch test. */ +void nxt_runtime_test_close_idle_connections(nxt_event_engine_t *engine); +#endif + + #endif /* _NXT_RUNTIME_H_INCLIDED_ */ diff --git a/src/test/nxt_conn_close_idle_test.c b/src/test/nxt_conn_close_idle_test.c new file mode 100644 index 000000000..a46dd2e07 --- /dev/null +++ b/src/test/nxt_conn_close_idle_test.c @@ -0,0 +1,125 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test for M-5: nxt_conn_accept_close_idle() (nxt_conn_accept.c) + * used to call nxt_fd_event_disable_read() unconditionally. When the 100ms + * listen timer fires and the spare conn still cannot be allocated, this + * function runs again on a listener that is already inactive; disabling an + * inactive epoll read is what turns into an epoll_ctl() ENOENT alert every + * 100ms (see nxt_epoll_engine.c). The fix guards the call with + * nxt_fd_event_is_active(), matching nxt_router_listen_socket_close(). + * + * nxt_conn_accept_close_idle() itself is static; nxt_conn_accept_error() is + * the one entry point into it, for the EMFILE/ENFILE/ENOBUFS/ENOMEM cases. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static nxt_uint_t nxt_conn_close_idle_test_disable_calls; + + +static void +nxt_conn_close_idle_test_disable_read(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + nxt_conn_close_idle_test_disable_calls++; + + /* What a real disable does: the read side becomes inactive. */ + ev->read = NXT_EVENT_INACTIVE; +} + + +nxt_int_t +nxt_conn_close_idle_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_listen_event_t lev; + nxt_event_engine_t engine, *saved_engine; + nxt_work_queue_cache_t cache; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn close idle test started"); + + ret = NXT_ERROR; + + task = thr->task; + task->thread = thr; + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&lev, sizeof(nxt_listen_event_t)); + + if (nxt_slow_path(nxt_timers_init(&engine.timers, 4) != NXT_OK)) { + nxt_log_alert(thr->log, "conn close idle test failed to init " + "timers"); + return NXT_ERROR; + } + + engine.event.disable_read = nxt_conn_close_idle_test_disable_read; + + nxt_work_queue_cache_create(&cache, 1024); + + engine.close_work_queue.cache = &cache; + nxt_work_queue_name(&engine.close_work_queue, "close"); + + /* nxt_conn_accept_close_idle() reads the engine off task->thread. */ + saved_engine = thr->engine; + thr->engine = &engine; + + lev.socket.task = task; + lev.socket.log = thr->log; + lev.socket.read = NXT_EVENT_DEFAULT; /* armed, as a live listener is */ + lev.timer.task = task; + lev.timer.log = thr->log; + + nxt_conn_close_idle_test_disable_calls = 0; + + /* + * First round: the listener is active, so the guard must still let the + * disable through -- this is not a "never disable" fix. + */ + nxt_conn_accept_error(task, &lev, "test", NXT_ENOMEM); + + if (nxt_slow_path(nxt_conn_close_idle_test_disable_calls != 1)) { + nxt_log_alert(thr->log, "conn close idle test: an active listener " + "was not disabled (%ui calls)", + nxt_conn_close_idle_test_disable_calls); + goto done; + } + + /* + * Second round: the 100ms timer fired again and the spare conn still + * cannot be allocated, so this runs again on the now-inactive listener. + * Before the fix, disable_read() would be called again here. + */ + nxt_conn_accept_error(task, &lev, "test", NXT_ENOMEM); + + if (nxt_slow_path(nxt_conn_close_idle_test_disable_calls != 1)) { + nxt_log_alert(thr->log, "conn close idle test: disable_read() was " + "called on an already inactive listener (%ui calls " + "total) -- this is the epoll_ctl ENOENT storm", + nxt_conn_close_idle_test_disable_calls); + goto done; + } + + ret = NXT_OK; + +done: + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_free(engine.timers.changes); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "conn close idle test passed"); + } + + return ret; +} diff --git a/src/test/nxt_conn_io_accept_test.c b/src/test/nxt_conn_io_accept_test.c new file mode 100644 index 000000000..44117d342 --- /dev/null +++ b/src/test/nxt_conn_io_accept_test.c @@ -0,0 +1,243 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test for M-4: when nxt_conn_io_accept() (nxt_conn_accept.c) + * cannot make an accepted socket non-blocking, it closes the socket and + * returns. The spare conn (lev->next) must then keep fd -1. Before the + * fix it kept the number of the closed socket, and nxt_conn_free() of the + * spare, at a listener release, asserts that the fd is -1. + * + * The test accepts a real TCP connection on 127.0.0.1 and makes the + * non-blocking step fail with the NXT_TESTS hook. It checks the spare fd, + * that the rejected socket was not passed to nxt_conn_accept(), and that + * the accepted socket was closed: the client reads end of file. + * + * Before the fix, the code went on into nxt_conn_accept(). The fixture + * engine, listen socket and work queue let that path run, so the test + * reports the failure instead of a crash. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static void +nxt_conn_io_accept_test_handler(nxt_task_t *task, void *obj, void *data) +{ +} + + +/* + * Before the fix, the code went on into nxt_conn_accept() and then into + * nxt_conn_accept_close_idle(), which can disable the listener. With this + * handler the M-4 test does not depend on the M-5 fix. + */ + +static void +nxt_conn_io_accept_test_disable_read(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + ev->read = NXT_EVENT_INACTIVE; +} + + +nxt_int_t +nxt_conn_io_accept_test(nxt_thread_t *thr) +{ +#if (NXT_LINUX) + int lfd, cfd; + char ch; + void *obj, *data; + size_t size; + ssize_t n; + nxt_int_t ret; + socklen_t len; + nxt_conn_t c; + nxt_task_t *t; + nxt_sockaddr_t *remote; + nxt_work_queue_t wq; + nxt_event_engine_t engine, *saved_engine; + nxt_work_handler_t handler; + nxt_listen_event_t lev; + struct sockaddr_in sin; + nxt_listen_socket_t ls; + nxt_work_queue_cache_t cache; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn io accept test started"); + + ret = NXT_ERROR; + cfd = -1; + remote = NULL; + + lfd = socket(AF_INET, SOCK_STREAM, 0); + if (lfd == -1) { + nxt_log_alert(thr->log, "conn io accept test: socket() failed %E", + nxt_errno); + return NXT_ERROR; + } + + /* + * The fixture for nxt_conn_accept(): an engine at max_connections, so + * no new spare conn is allocated, a listen socket with a handler that + * does nothing, and a work queue. + */ + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + + if (nxt_timers_init(&engine.timers, 4) != NXT_OK) { + (void) close(lfd); + return NXT_ERROR; + } + + engine.task = *thr->task; + engine.event.disable_read = nxt_conn_io_accept_test_disable_read; + nxt_queue_init(&engine.idle_connections); + nxt_queue_init(&engine.active_connections); + + nxt_work_queue_cache_create(&cache, 0); + + engine.close_work_queue.cache = &cache; + + nxt_memzero(&wq, sizeof(nxt_work_queue_t)); + wq.cache = &cache; + + nxt_memzero(&ls, sizeof(nxt_listen_socket_t)); + ls.handler = nxt_conn_io_accept_test_handler; + + saved_engine = thr->engine; + thr->engine = &engine; + + /* The cleanup at "done" reads c and lev, also after an early failure. */ + + nxt_memzero(&c, sizeof(nxt_conn_t)); + c.socket.fd = -1; + + nxt_memzero(&lev, sizeof(nxt_listen_event_t)); + lev.timer.task = &engine.task; + lev.timer.log = thr->log; + + nxt_memzero(&sin, sizeof(struct sockaddr_in)); + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + len = sizeof(struct sockaddr_in); + + if (bind(lfd, (struct sockaddr *) &sin, len) != 0 + || listen(lfd, 1) != 0 + || getsockname(lfd, (struct sockaddr *) &sin, &len) != 0) + { + nxt_log_alert(thr->log, "conn io accept test: listen failed %E", + nxt_errno); + goto done; + } + + cfd = socket(AF_INET, SOCK_STREAM, 0); + if (cfd == -1 + || connect(cfd, (struct sockaddr *) &sin, len) != 0) + { + nxt_log_alert(thr->log, "conn io accept test: connect failed %E", + nxt_errno); + goto done; + } + + /* Room for the address text that nxt_conn_accept() writes. */ + size = offsetof(nxt_sockaddr_t, u) + sizeof(struct sockaddr_in) + + NXT_INET_ADDR_STR_LEN; + + remote = nxt_zalloc(size); + if (remote == NULL) { + goto done; + } + + remote->socklen = sizeof(struct sockaddr_in); + remote->length = NXT_INET_ADDR_STR_LEN; + + c.remote = remote; + + lev.socket.fd = lfd; + lev.listen = &ls; + lev.work_queue = &wq; + lev.next = &c; + lev.ready = 1; + + nxt_conn_test_nonblocking_failures = 1; + + nxt_conn_io_accept(thr->task, &lev, NULL); + + if (nxt_conn_test_nonblocking_failures != 0) { + nxt_log_alert(thr->log, "conn io accept test failed: the " + "non-blocking step was not reached"); + goto done; + } + + if (wq.head != NULL || c.listen != NULL) { + nxt_log_alert(thr->log, "conn io accept test failed: the rejected " + "socket was passed to nxt_conn_accept()"); + goto done; + } + + if (c.socket.fd != -1) { + nxt_log_alert(thr->log, "conn io accept test failed: the spare conn " + "kept fd %d of a closed socket", c.socket.fd); + goto done; + } + + /* The accepted socket is closed: the client reads end of file. */ + + n = read(cfd, &ch, 1); + if (n != 0) { + nxt_log_alert(thr->log, "conn io accept test failed: the accepted " + "socket was not closed (read %z)", n); + goto done; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "conn io accept test passed"); + + ret = NXT_OK; + +done: + + nxt_conn_test_nonblocking_failures = 0; + + /* Undo what nxt_conn_accept() did, if it ran. */ + + nxt_conn_untrack(&engine, (&c)); + + nxt_timer_delete(&engine, &lev.timer); + + while (wq.head != NULL) { + (void) nxt_work_queue_pop(&wq, &t, &obj, &data); + } + + while (engine.close_work_queue.head != NULL) { + handler = nxt_work_queue_pop(&engine.close_work_queue, &t, &obj, + &data); + handler(t, obj, data); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_free(engine.timers.changes); + + if (remote != NULL) { + nxt_free(remote); + } + + if (cfd != -1) { + (void) close(cfd); + } + + (void) close(lfd); + + return ret; + +#else + + return NXT_OK; + +#endif +} diff --git a/src/test/nxt_runtime_idle_close_test.c b/src/test/nxt_runtime_idle_close_test.c new file mode 100644 index 000000000..59f86d663 --- /dev/null +++ b/src/test/nxt_runtime_idle_close_test.c @@ -0,0 +1,463 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test for M-11: nxt_runtime_close_idle_connections() + * (src/nxt_runtime.c) must close every idle conn through the conn's own + * protocol -- c->read_state->close_handler, the contract + * nxt_conn_accept_close_idle_handler() already uses under fd pressure -- + * and must leave a conn that has no protocol state yet alone. + * + * Before this fix, the idle-close pass called nxt_conn_close() directly: + * + * - a keep-alive h1p conn still carries nxt_h1p_request_send_state, + * whose ready_handler (nxt_h1p_conn_sent) is a no-op with c->write == + * NULL -- the conn was closed but never freed, a leak (CONN-INV-2); + * + * - a conn that never received a request has write_state == NULL, and + * nxt_conn_close_handler() dereferenced it, i.e. crashed; + * + * - a generic release (just nxt_conn_free()) is not enough either: the + * protocol free handlers also return c->remote to the engine's sockaddr + * cache (nxt_conn_accept_alloc() took it from there) and, for a router + * conn, drop the listener reference taken in nxt_conn_accept(), without + * which a draining listener never reaches its final release. + * + * The real protocol states (nxt_h1p_keepalive_state, + * nxt_controller_conn_read_state) are private to their modules and need a + * live listener configuration, so the protocol-owned conn below carries a + * stand-in read_state whose close_handler does what nxt_h1p_conn_close() + * ends up doing -- install a close state and nxt_conn_close() -- and whose + * close state's ready_handler does exactly what nxt_h1p_conn_free() does: + * nxt_sockaddr_cache_free(), nxt_conn_free() and the real + * nxt_router_listen_event_release(). What the test asserts is the + * dispatch contract of the idle-close pass: the protocol close handler is + * what runs, to completion, and the not-yet-initialized conn is untouched. + */ + +#include +#include +#include +#include +#include +#include +#include "nxt_tests.h" + +#include + + +static nxt_uint_t nxt_runtime_idle_close_test_stale_calls; +static nxt_uint_t nxt_runtime_idle_close_test_close_calls; +static nxt_uint_t nxt_runtime_idle_close_test_free_calls; +static nxt_conn_t *nxt_runtime_idle_close_test_closed_conn; + + +static void +nxt_runtime_idle_close_test_stale_handler(nxt_task_t *task, void *obj, + void *data) +{ + /* + * Stands in for nxt_h1p_conn_sent(): the leftover request-send + * ready_handler of a keep-alive conn, which does nothing to the conn + * itself. If this runs, the idle-close pass bypassed the protocol and + * closed the conn with its stale write_state. + */ + nxt_runtime_idle_close_test_stale_calls++; +} + + +static const nxt_conn_state_t nxt_runtime_idle_close_test_stale_state + nxt_aligned(64) = +{ + .ready_handler = nxt_runtime_idle_close_test_stale_handler, +}; + + +static void +nxt_runtime_idle_close_test_proto_free(nxt_task_t *task, void *obj, + void *data) +{ + nxt_conn_t *c; + nxt_listen_event_t *lev; + nxt_event_engine_t *engine; + + c = obj; + + /* The body of nxt_h1p_conn_free(). */ + + nxt_runtime_idle_close_test_free_calls++; + + engine = task->thread->engine; + + nxt_sockaddr_cache_free(engine, c); + + lev = c->listen; + + nxt_conn_free(task, c); + + nxt_router_listen_event_release(&engine->task, lev, NULL); +} + + +static const nxt_conn_state_t nxt_runtime_idle_close_test_proto_close_state + nxt_aligned(64) = +{ + .ready_handler = nxt_runtime_idle_close_test_proto_free, +}; + + +static void +nxt_runtime_idle_close_test_proto_close(nxt_task_t *task, void *obj, + void *data) +{ + nxt_conn_t *c; + + c = obj; + + /* What nxt_h1p_conn_close() ends up doing (nxt_h1p_conn_closing()). */ + + nxt_runtime_idle_close_test_close_calls++; + nxt_runtime_idle_close_test_closed_conn = c; + + c->write_state = &nxt_runtime_idle_close_test_proto_close_state; + + nxt_conn_close(task->thread->engine, c); +} + + +static const nxt_conn_state_t nxt_runtime_idle_close_test_proto_read_state + nxt_aligned(64) = +{ + .close_handler = nxt_runtime_idle_close_test_proto_close, +}; + + +/* + * A minimal event interface: only .close is exercised by the close path + * this test drives (c->socket.error is set to 1 below, so nxt_conn_close() + * takes the close_work_queue branch directly and never touches shutdown, + * enable/disable or poll). Returning 0 tells nxt_conn_close_handler() that + * nothing is pending, so it closes the fd and dispatches write_state's + * ready_handler on the same pass -- no timer round trip to wait out. + */ +static nxt_bool_t +nxt_runtime_idle_close_test_event_close(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + return 0; +} + + +static void +nxt_runtime_idle_close_test_drain(nxt_work_queue_t *wq) +{ + void *obj, *data; + nxt_task_t *task; + nxt_work_handler_t handler; + + while (wq->head != NULL) { + handler = nxt_work_queue_pop(wq, &task, &obj, &data); + handler(task, obj, data); + } +} + + +nxt_int_t +nxt_runtime_idle_close_test(nxt_thread_t *thr) +{ + int fds[2]; + nxt_mp_t *mp; + nxt_int_t ret; + nxt_conn_t *proto, *fresh; + nxt_task_t *task; + nxt_sockaddr_t sa, *remote; + nxt_listen_event_t lev; + nxt_listen_socket_t ls; + nxt_event_engine_t engine, *saved_engine; + nxt_work_queue_cache_t cache; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "runtime idle close test started"); + + ret = NXT_ERROR; + fds[0] = -1; + fds[1] = -1; + proto = NULL; + fresh = NULL; + + task = thr->task; + task->thread = thr; + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&lev, sizeof(nxt_listen_event_t)); + nxt_memzero(&ls, sizeof(nxt_listen_socket_t)); + nxt_memzero(&sa, sizeof(nxt_sockaddr_t)); + + /* Backs the conn structs and the sockaddr cache. */ + engine.mem_pool = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(engine.mem_pool == NULL)) { + return NXT_ERROR; + } + + engine.task = *task; + engine.event.close = nxt_runtime_idle_close_test_event_close; + + nxt_queue_init(&engine.idle_connections); + nxt_queue_init(&engine.active_connections); + nxt_queue_init(&engine.listen_connections); + nxt_queue_init(&engine.joints); + + nxt_work_queue_cache_create(&cache, 1024); + + engine.fast_work_queue.cache = &cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + engine.close_work_queue.cache = &cache; + nxt_work_queue_name(&engine.close_work_queue, "close"); + + engine.shutdown_work_queue.cache = &cache; + nxt_work_queue_name(&engine.shutdown_work_queue, "shutdown"); + + /* Enough of a listen socket to size and type cached sockaddrs from. */ + sa.type = SOCK_STREAM; + sa.u.sockaddr.sa_family = AF_INET; + + ls.socket = -1; + ls.sockaddr = &sa; + ls.socklen = sizeof(struct sockaddr_in); + ls.address_length = NXT_INET_ADDR_STR_LEN; + + /* + * The listener the conns were accepted on: its own reference plus the + * one nxt_conn_accept() takes for the protocol-owned conn below. Not + * draining and without a joint, so nxt_router_listen_event_release() + * only drops the count. + */ + lev.listen = &ls; + lev.count = 2; + + saved_engine = thr->engine; + thr->engine = &engine; + + /* + * Real, valid fds so nxt_socket_close() in the close handler closes + * something real rather than an already-invalid descriptor. + */ + if (nxt_slow_path(pipe(fds) != 0)) { + nxt_log_alert(thr->log, "runtime idle close test failed to open a " + "pipe"); + goto done; + } + + /* + * A keep-alive conn owned by its protocol: as nxt_conn_accept_alloc() + * and nxt_conn_accept() leave it, with a cached sockaddr, a listener + * reference, and a stale write_state from its last response. + */ + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + goto done; + } + + proto = nxt_conn_create(mp, task); + if (nxt_slow_path(proto == NULL)) { + nxt_mp_destroy(mp); + goto done; + } + + /* From here on the per-conn pool belongs to the conn. */ + + proto->socket.fd = fds[0]; + fds[0] = -1; + + proto->remote = nxt_sockaddr_cache_alloc(&engine, &ls); + if (nxt_slow_path(proto->remote == NULL)) { + goto done; + } + + remote = proto->remote; + + proto->listen = &lev; + + /* Bypasses the shutdown() syscall branch in nxt_conn_close(). */ + proto->socket.error = 1; + + proto->read_state = &nxt_runtime_idle_close_test_proto_read_state; + proto->write_state = &nxt_runtime_idle_close_test_stale_state; + + /* Idle, and eligible for the close pass (read_ready == 0). */ + proto->socket.read_ready = 0; + nxt_conn_idle(&engine, proto); + + /* + * A conn accepted a moment ago whose listen handler has not run yet: + * no protocol state at all. Before the fix the pass closed it with + * write_state == NULL, a crash in nxt_conn_close_handler(); the pass + * must leave it alone now. + */ + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + goto done; + } + + fresh = nxt_conn_create(mp, task); + if (nxt_slow_path(fresh == NULL)) { + nxt_mp_destroy(mp); + goto done; + } + + fresh->socket.fd = fds[1]; + fds[1] = -1; + + fresh->remote = nxt_sockaddr_cache_alloc(&engine, &ls); + if (nxt_slow_path(fresh->remote == NULL)) { + goto done; + } + + fresh->listen = &lev; + fresh->socket.error = 1; + fresh->socket.read_ready = 0; + nxt_conn_idle(&engine, fresh); + + nxt_runtime_idle_close_test_stale_calls = 0; + nxt_runtime_idle_close_test_close_calls = 0; + nxt_runtime_idle_close_test_free_calls = 0; + nxt_runtime_idle_close_test_closed_conn = NULL; + + nxt_runtime_test_close_idle_connections(&engine); + + /* Runs nxt_conn_close_handler(), which dispatches to fast_work_queue. */ + nxt_runtime_idle_close_test_drain(&engine.close_work_queue); + nxt_runtime_idle_close_test_drain(&engine.fast_work_queue); + + if (nxt_slow_path(nxt_runtime_idle_close_test_close_calls != 1 + || nxt_runtime_idle_close_test_closed_conn != proto)) + { + nxt_log_alert(thr->log, "runtime idle close test: the protocol " + "close handler did not run exactly once for the " + "protocol-owned conn (%ui calls)", + nxt_runtime_idle_close_test_close_calls); + goto done; + } + + if (nxt_slow_path(nxt_runtime_idle_close_test_stale_calls != 0)) { + nxt_log_alert(thr->log, "runtime idle close test: the stale " + "write_state ran -- the idle-close pass bypassed the " + "protocol close handler"); + goto done; + } + + if (nxt_slow_path(nxt_runtime_idle_close_test_free_calls != 1)) { + nxt_log_alert(thr->log, "runtime idle close test: the protocol " + "free handler did not run (%ui calls)", + nxt_runtime_idle_close_test_free_calls); + goto done; + } + + /* The protocol free path ran: proto is gone, only fresh is left. */ + proto = NULL; + + if (nxt_slow_path(lev.count != 1)) { + nxt_log_alert(thr->log, "runtime idle close test: the listener " + "reference was not released (count %uD, expected 1)", + lev.count); + goto done; + } + + /* + * The sockaddr went back to the engine cache: the cache hands out its + * free list first, so the next allocation for this listener must be + * that very block (it lives in the engine pool, destroyed below). + */ + if (nxt_slow_path(nxt_sockaddr_cache_alloc(&engine, &ls) != remote)) { + nxt_log_alert(thr->log, "runtime idle close test: the closed " + "conn's sockaddr was not returned to the engine " + "cache"); + goto done; + } + + if (nxt_slow_path(engine.connections != 1)) { + nxt_log_alert(thr->log, "runtime idle close test: engine accounts " + "for %uD connection(s), expected 1 -- either the " + "protocol-owned conn was not freed or the " + "uninitialized one was", engine.connections); + goto done; + } + + if (nxt_slow_path(nxt_queue_is_empty(&engine.idle_connections) + || nxt_queue_first(&engine.idle_connections) + != &fresh->link + || nxt_queue_next(&fresh->link) + != nxt_queue_tail(&engine.idle_connections) + || fresh->idle != NXT_CONN_TRACK_IDLE)) + { + nxt_log_alert(thr->log, "runtime idle close test: idle_connections " + "must hold exactly the not-yet-initialized conn"); + goto done; + } + + if (nxt_slow_path(fresh->write_state != NULL + || fresh->socket.fd == -1)) + { + nxt_log_alert(thr->log, "runtime idle close test: the " + "not-yet-initialized conn was touched by the pass"); + goto done; + } + + ret = NXT_OK; + +done: + + /* What the listen handler and the protocol would eventually do. */ + + if (fresh != NULL) { + if (fresh->socket.fd != -1) { + (void) close(fresh->socket.fd); + fresh->socket.fd = -1; + } + + nxt_conn_untrack(&engine, fresh); + + if (fresh->remote != NULL) { + nxt_sockaddr_cache_free(&engine, fresh); + } + + nxt_conn_free(task, fresh); + } + + if (proto != NULL) { + if (proto->socket.fd != -1) { + (void) close(proto->socket.fd); + proto->socket.fd = -1; + } + + nxt_conn_untrack(&engine, proto); + + if (proto->remote != NULL) { + nxt_sockaddr_cache_free(&engine, proto); + } + + nxt_conn_free(task, proto); + } + + if (fds[0] != -1) { + (void) close(fds[0]); + } + + if (fds[1] != -1) { + (void) close(fds[1]); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_mp_destroy(engine.mem_pool); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, + "runtime idle close test passed"); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 9efd89b4d..8a9700733 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -281,6 +281,19 @@ main(int argc, char **argv) if (nxt_port_ready_test(thr) != NXT_OK) { return 1; } + + if (nxt_conn_close_idle_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_conn_io_accept_test(thr) != NXT_OK) { + return 1; + } + + if (nxt_runtime_idle_close_test(thr) != NXT_OK) { + return 1; + } + if (nxt_router_new_port_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index c16f912c0..c54d7260b 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -80,6 +80,9 @@ nxt_int_t nxt_port_mmap_range_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmap_read_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmaps_max_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); +nxt_int_t nxt_conn_close_idle_test(nxt_thread_t *thr); +nxt_int_t nxt_conn_io_accept_test(nxt_thread_t *thr); +nxt_int_t nxt_runtime_idle_close_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_test(nxt_thread_t *thr); nxt_int_t nxt_router_start_fail_soak_test(nxt_thread_t *thr); From 1acaf8032808cec035958383d069cb9dd70dc94c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 01:31:01 +0000 Subject: [PATCH 330/391] fix(router): keep a listener whose spare conn or listen event fails M-6 from the audit of the nxt_conn_t state machine. When the spare conn could not be allocated, nxt_listen_event() returned a listener that was not armed and not linked, so it never accepted. Returning NULL instead is not safe: the router NULL branch released a listen socket reference that this engine did not take (the count could underflow, or reach 0 and close a shared fd under another engine), and it did not acknowledge the router job, so the configuration request did not complete. On master this already happens when the lev allocation itself fails. So: - nxt_listen_event() keeps the listener when only the spare conn fails. The listener is linked but not armed, and goes to nxt_conn_accept_close_idle(), the recovery that the accept path already uses. The 100 ms timer retries the spare and arms accept when it gets one. The nxt_fd_event_is_active() check there keeps its disable_read() from acting on this listener that was never armed. - nxt_conn_accept_close_idle() queues the idle-close pass with the engine task. The task of the caller can be the task of a router job, which is freed with the configuration after the job is posted. - The timer-driven first accept starts a batch (lev->ready = lev->batch), as nxt_conn_listen_handler() does. Before, lev->ready was 0 from nxt_zalloc() and wrapped. Router engines have batch 0, so for them nothing changes. - The alert "new connections are not accepted within 100ms" comes only when an armed listener stops accepting, not for a new listener that was never armed, and not at each 100 ms retry. - The idle-close pass skips a conn without read_state or close_handler, as nxt_runtime_close_idle_connections() does. This is not reachable today, because the accept-init work of a conn runs before the close queue. - nxt_listen_event() returns NULL only when the lev allocation fails. The router NULL branch no longer drops a reference, and it acknowledges the job. - The create job takes its reference on the listen socket before nxt_listen_event(), so an engine whose listen event allocation failed also holds one. Its joint also stays linked, with the stale flag: releasing it in the create job could drop the last reference to the configuration that is being applied. The release is deferred to the next job for this listener; until then, the joint keeps the configuration and the listen socket. - The update and delete handlers accept a missing lev on the engine: update creates it, and delete acknowledges. Both release the stale joints and their listen socket references, with nxt_router_listen_socket_release_stale(); an update after the new create, which took its own reference. So the old configurations are freed, a worker thread that has to quit can exit, and the last reference closes the listen socket also when no engine could allocate the listen event (before, the address stayed bound). A delete releases before it acknowledges, so the acknowledgement means that the port is free, as after nxt_router_listen_socket_close_finish(). - Only joints with the stale flag are released, matched by the listen socket and not by the socket conf: when an engine is removed, the delete job has the socket conf of the new configuration, and the stale joint has the old one. The flag keeps a joint that a request holds for a closed listener from matching a new listen socket at the same address. Tests: - nxt_listen_event_test.c checks both halves of the listener recovery, that the idle-close pass has the engine task, and that the first accept after the retry gets a full batch. - nxt_router_stale_joint_test.c checks the stale joint and listen socket release, a joint without the stale flag, and the last reference of a real socket. It also runs the real create and update jobs, with the NXT_TESTS hook nxt_listen_event_test_alloc_failures: the create keeps its joint with the flag, takes one listen socket reference and acknowledges the job; the update creates the listen event, releases the stale joint and acknowledges. - Each of these mutations makes a test fail: no close_idle call on a missing spare, no listen socket release in the stale release, no ls->count++ before the listen event, no stale release in the update, no stale flag check, the caller task on the idle-close pass, and no batch before the timer-driven accept. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 8 + auto/sources | 2 + docs/changes.xml | 11 + src/nxt_conn.h | 1 + src/nxt_conn_accept.c | 76 +++- src/nxt_router.c | 169 +++++++- src/nxt_router.h | 17 + src/test/nxt_listen_event_test.c | 321 +++++++++++++++ src/test/nxt_router_stale_joint_test.c | 547 +++++++++++++++++++++++++ src/test/nxt_tests.c | 8 + src/test/nxt_tests.h | 2 + 11 files changed, 1147 insertions(+), 15 deletions(-) create mode 100644 src/test/nxt_listen_event_test.c create mode 100644 src/test/nxt_router_stale_joint_test.c diff --git a/CHANGES b/CHANGES index 859348645..88568671d 100644 --- a/CHANGES +++ b/CHANGES @@ -96,6 +96,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 directory)" alert every 100 milliseconds until a connection was freed. + *) Bugfix: when a router thread could not allocate the listen event + while a listener was added, a configuration request could wait + forever, or a listening socket shared by several router threads + could be closed too early. When the thread had no free connection + slot instead, the listener never accepted connections. Now the + request completes, and a listener without a free slot starts to + accept connections when a slot is free. + *) Bugfix: at shutdown, idle client connections were closed but not freed. They are now closed through their protocol handlers. diff --git a/auto/sources b/auto/sources index 1bb4ac920..bf7c7e011 100644 --- a/auto/sources +++ b/auto/sources @@ -191,6 +191,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_mmaps_max_test.c \ src/test/nxt_port_ready_test.c \ src/test/nxt_conn_close_idle_test.c \ + src/test/nxt_listen_event_test.c \ src/test/nxt_conn_io_accept_test.c \ src/test/nxt_runtime_idle_close_test.c \ src/test/nxt_router_new_port_test.c \ @@ -204,6 +205,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_remove_pid_soak_test.c \ src/test/nxt_router_detached_test.c \ src/test/nxt_router_sender_test.c \ + src/test/nxt_router_stale_joint_test.c \ src/test/nxt_router_websocket_test.c \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index 81ad22212..b2bb20650 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -52,6 +52,17 @@ milliseconds until a connection was freed. + + +when a router thread could not allocate the listen event while a listener was +added, a configuration request could wait forever, or a listening socket +shared by several router threads could be closed too early. When the thread +had no free connection slot instead, the listener never accepted connections. +Now the request completes, and a listener without a free slot starts to accept +connections when a slot is free. + + + at shutdown, idle client connections were closed but not freed. They are now diff --git a/src/nxt_conn.h b/src/nxt_conn.h index 42ee62c28..4df52c053 100644 --- a/src/nxt_conn.h +++ b/src/nxt_conn.h @@ -268,6 +268,7 @@ void nxt_conn_io_accept(nxt_task_t *task, void *obj, void *data); #if (NXT_TESTS) extern nxt_uint_t nxt_conn_test_nonblocking_failures; +extern nxt_uint_t nxt_listen_event_test_alloc_failures; #endif NXT_EXPORT void nxt_conn_accept(nxt_task_t *task, nxt_listen_event_t *lev, diff --git a/src/nxt_conn_accept.c b/src/nxt_conn_accept.c index 6efa3ce37..d27a23eb6 100644 --- a/src/nxt_conn_accept.c +++ b/src/nxt_conn_accept.c @@ -34,12 +34,27 @@ static void nxt_conn_listen_timer_handler(nxt_task_t *task, void *obj, void *data); +#if (NXT_TESTS) + +/* For src/test/nxt_router_stale_joint_test.c: fail the next N allocations. */ +nxt_uint_t nxt_listen_event_test_alloc_failures; + +#endif + + nxt_listen_event_t * nxt_listen_event(nxt_task_t *task, nxt_listen_socket_t *ls) { nxt_listen_event_t *lev; nxt_event_engine_t *engine; +#if (NXT_TESTS) + if (nxt_listen_event_test_alloc_failures > 0) { + nxt_listen_event_test_alloc_failures--; + return NULL; + } +#endif + lev = nxt_zalloc(sizeof(nxt_listen_event_t)); if (nxt_fast_path(lev != NULL)) { @@ -69,16 +84,29 @@ nxt_listen_event(nxt_task_t *task, nxt_listen_socket_t *ls) lev->socket.task = &lev->task; lev->timer.task = &lev->task; - if (nxt_conn_accept_alloc(task, lev) != NULL) { + nxt_queue_insert_tail(&engine->listen_connections, &lev->link); + + if (nxt_fast_path(nxt_conn_accept_alloc(task, lev) != NULL)) { nxt_fd_event_enable_accept(engine, &lev->socket); - nxt_queue_insert_tail(&engine->listen_connections, &lev->link); + } else { + /* + * The spare conn could not be allocated (the engine is at + * max_connections, or memory is short). The listener is + * still valid: it is linked above, so the router can find it + * for later updates and closes, and it is simply not armed + * yet. Treat this exactly like the same failure on the accept + * path (nxt_conn_accept_next()): close idle conns to make room + * and let the 100ms listen timer retry the spare, which arms + * accept once it succeeds (nxt_conn_listen_timer_handler()). + * The nxt_fd_event_is_active() check in there keeps its + * disable_read() from acting on this never-armed listener. + */ + nxt_conn_accept_close_idle(task, lev); } - - return lev; } - return NULL; + return lev; } @@ -294,16 +322,31 @@ nxt_conn_accept_close_idle(nxt_task_t *task, nxt_listen_event_t *lev) engine = task->thread->engine; + /* + * The work runs later, so it gets the engine task. The task of the + * caller can be temporary: nxt_listen_event() runs in a router job, + * whose task is freed with the configuration after the job is posted. + */ nxt_work_queue_add(&engine->close_work_queue, - nxt_conn_accept_close_idle_handler, task, NULL, NULL); + nxt_conn_accept_close_idle_handler, &engine->task, + NULL, NULL); nxt_timer_add(engine, &lev->timer, 100); + /* + * Alert only when an armed listener stops accepting. The timer retries + * of the same stall, and a new listener that was never armed, come + * here with the read side inactive. + */ if (nxt_fd_event_is_active(lev->socket.read)) { nxt_fd_event_disable_read(engine, &lev->socket); - } - nxt_alert(task, "new connections are not accepted within 100ms"); + nxt_alert(task, "new connections are not accepted within 100ms"); + + } else { + nxt_debug(task, "listener %d: no spare conn, retry in 100ms", + lev->socket.fd); + } } @@ -335,6 +378,16 @@ nxt_conn_accept_close_idle_handler(nxt_task_t *task, void *obj, void *data) nxt_debug(c->socket.task, "idle connection: %d rdy:%d", c->socket.fd, c->socket.read_ready); + /* + * Not reached today: the accept-init work of a conn is queued + * before this pass, and the close queue runs after the read and + * write queues. Checked as nxt_runtime_close_idle_connections() + * does, so a later change of the queue order cannot crash here. + */ + if (c->read_state == NULL || c->read_state->close_handler == NULL) { + continue; + } + /* * Close regardless of read_ready: a pending FIN from the peer * (CLOSE-WAIT) sets read_ready = 1 but the connection is still @@ -413,6 +466,13 @@ nxt_conn_listen_timer_handler(nxt_task_t *task, void *obj, void *data) nxt_fd_event_enable_accept(task->thread->engine, &lev->socket); + /* + * Start a new batch, as nxt_conn_listen_handler() does. A listener + * that was never armed has ready 0 from nxt_zalloc() here, and the + * accept would wrap it. + */ + lev->ready = lev->batch; + lev->accept(task, lev, c); } diff --git a/src/nxt_router.c b/src/nxt_router.c index 97098ce02..a8c809c40 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -341,6 +341,9 @@ static void nxt_router_listen_socket_create(nxt_task_t *task, void *obj, void *data); static void nxt_router_listen_socket_update(nxt_task_t *task, void *obj, void *data); +static void nxt_router_listen_socket_release_stale(nxt_task_t *task, + nxt_event_engine_t *engine, nxt_listen_socket_t *ls, + nxt_socket_conf_joint_t *keep); static void nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data); static void nxt_router_worker_thread_quit(nxt_task_t *task, void *obj, @@ -4937,6 +4940,7 @@ nxt_router_engine_joints_create(nxt_router_temp_conf_t *tmcf, } joint->count = 1; + joint->stale = 0; skcf = nxt_queue_link_data(qlk, nxt_socket_conf_t, link); skcf->count++; @@ -5248,20 +5252,45 @@ nxt_router_listen_socket_create(nxt_task_t *task, void *obj, void *data) skcf = joint->socket_conf; ls = skcf->listen; + /* + * This engine holds one reference on the listen socket, also when the + * listen event cannot be allocated below. The last engine that + * releases its reference closes the socket, see + * nxt_router_listen_socket_release(). + */ + lock = &skcf->router_conf->router->lock; + + nxt_thread_spin_lock(lock); + ls->count++; + nxt_thread_spin_unlock(lock); + lev = nxt_listen_event(task, ls); if (nxt_slow_path(lev == NULL)) { - nxt_router_listen_socket_release(task, skcf); + /* + * Only the lev allocation itself can fail here: a missing spare + * conn keeps the listener and is retried by its own timer + * (nxt_listen_event()). Acknowledge the job so the configuration + * request completes instead of waiting forever. The joint and + * the listen socket reference stay: releasing the joint here could + * drop the last reference to the configuration that is being + * applied. A later update or delete of this listener releases + * both, see nxt_router_listen_socket_release_stale(). The release + * is deferred, not skipped: until that job, the stale joint keeps + * this configuration (its memory pool and TLS contexts) and the + * listen socket, so the address stays bound. + */ + nxt_alert(task, "engine %p: listen socket %d: no memory for the " + "listen event, no connections will be accepted on it", + task->thread->engine, ls->socket); + + joint->stale = 1; + + nxt_router_conf_wait_post(job); return; } lev->socket.data = joint; - lock = &skcf->router_conf->router->lock; - - nxt_thread_spin_lock(lock); - ls->count++; - nxt_thread_spin_unlock(lock); - nxt_router_conf_wait_post(job); } @@ -5309,6 +5338,30 @@ nxt_router_listen_socket_update(nxt_task_t *task, void *obj, void *data) lev = nxt_router_listen_event(&engine->listen_connections, joint->socket_conf); + if (nxt_slow_path(lev == NULL)) { + /* + * The previous configuration could not allocate the listen event + * on this engine (nxt_router_listen_socket_create()), so there is + * nothing to update: create it now instead. The create path + * links the joint itself, so unlink it again first. + */ + nxt_queue_remove(&joint->link); + + nxt_router_listen_socket_create(task, obj, data); + + /* + * The job is posted: its task and data can be freed now, so the + * engine task is used below. An update keeps the listen socket, + * so the stale joint of the failed create has the same one. The + * create above took a new reference on it, so this release does + * not close it. + */ + nxt_router_listen_socket_release_stale(&engine->task, engine, + joint->socket_conf->listen, + joint); + return; + } + old = lev->socket.data; lev->socket.data = joint; lev->listen = joint->socket_conf->listen; @@ -5338,6 +5391,30 @@ nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data) lev = nxt_router_listen_event(&engine->listen_connections, skcf); + if (nxt_slow_path(lev == NULL)) { + /* + * The listen event was never allocated on this engine + * (nxt_router_listen_socket_create()), so there is no listener to + * close. Release the stale joint and its listen socket reference + * first: if this was the last reference, the socket is closed, and + * the acknowledgement then means that the port is free, as after + * nxt_router_listen_socket_close_finish(). The listen socket is + * matched, not skcf: when an engine is removed + * (nxt_router_engine_conf_delete()), skcf is the socket conf of + * the new configuration, and the stale joint has the old one. + */ + nxt_router_listen_socket_release_stale(&engine->task, engine, + skcf->listen, NULL); + + nxt_router_conf_wait_post(obj); + + if (engine->shutdown && nxt_queue_is_empty(&engine->joints)) { + nxt_router_worker_thread_exit(&engine->task); + } + + return; + } + nxt_fd_event_delete(engine, &lev->socket); nxt_debug(task, "engine %p: listen socket delete: %d", engine, @@ -5353,6 +5430,84 @@ nxt_router_listen_socket_delete(nxt_task_t *task, void *obj, void *data) } +/* + * Releases the joints that a failed nxt_router_listen_socket_create() left + * in engine->joints for the listen socket ls, and the listen socket + * reference that each of them holds. Such a joint has the stale flag. + * Only the flag is trusted, not ls alone: a request can hold the joint of + * a closed listener in engine->joints, and a new listen socket can have + * the address of the freed one. "keep" is the joint of the current job. + * Each update or delete of ls releases its stale joints, so an engine has + * at most one of them for ls when this runs; the loop does not depend on + * that. The order is the same as in nxt_router_listen_socket_close_finish(): + * first the listen socket, which the last reference closes, then the + * joint. + */ + +static void +nxt_router_listen_socket_release_stale(nxt_task_t *task, + nxt_event_engine_t *engine, nxt_listen_socket_t *ls, + nxt_socket_conf_joint_t *keep) +{ + nxt_queue_link_t *qlk, *next; + nxt_socket_conf_joint_t *joint; + + for (qlk = nxt_queue_first(&engine->joints); + qlk != nxt_queue_tail(&engine->joints); + qlk = next) + { + next = nxt_queue_next(qlk); + + joint = nxt_queue_link_data(qlk, nxt_socket_conf_joint_t, link); + + if (joint == keep) { + continue; + } + + if (joint->stale && joint->socket_conf->listen == ls) { + nxt_debug(task, "engine %p: release stale joint %p", engine, + joint); + + nxt_router_listen_socket_release(task, joint->socket_conf); + nxt_router_conf_release(task, joint); + } + } +} + + +#if (NXT_TESTS) + +/* For src/test/nxt_router_stale_joint_test.c. */ + +void +nxt_router_test_listen_socket_release_stale(nxt_task_t *task, + nxt_event_engine_t *engine, nxt_listen_socket_t *ls, + nxt_socket_conf_joint_t *keep) +{ + nxt_router_listen_socket_release_stale(task, engine, ls, keep); +} + + +/* The listen socket jobs, for src/test/nxt_router_stale_joint_test.c. */ + +void +nxt_router_test_listen_socket_create(nxt_task_t *task, nxt_joint_job_t *job, + nxt_socket_conf_joint_t *joint) +{ + nxt_router_listen_socket_create(task, job, joint); +} + + +void +nxt_router_test_listen_socket_update(nxt_task_t *task, nxt_joint_job_t *job, + nxt_socket_conf_joint_t *joint) +{ + nxt_router_listen_socket_update(task, job, joint); +} + +#endif + + static void nxt_router_worker_thread_quit(nxt_task_t *task, void *obj, void *data) { diff --git a/src/nxt_router.h b/src/nxt_router.h index 8f475cb72..d886dad1f 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -317,6 +317,12 @@ typedef struct { nxt_joint_job_t *close_job; + /* + * The create job of this joint could not allocate the listen event, + * see nxt_router_listen_socket_release_stale(). + */ + uint8_t stale; /* 1 bit */ + nxt_upstream_t **upstreams; /* Modules configuraitons. */ @@ -387,6 +393,17 @@ nxt_router_temp_conf_t *nxt_router_test_temp_conf(nxt_task_t *task); void nxt_router_test_app_rpc_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_app_t *app); +/* The release of stale listen joints, for the stale joint test. */ +void nxt_router_test_listen_socket_release_stale(nxt_task_t *task, + nxt_event_engine_t *engine, nxt_listen_socket_t *ls, + nxt_socket_conf_joint_t *keep); + +/* The listen socket create and update jobs, for the stale joint test. */ +void nxt_router_test_listen_socket_create(nxt_task_t *task, + nxt_joint_job_t *job, nxt_socket_conf_joint_t *joint); +void nxt_router_test_listen_socket_update(nxt_task_t *task, + nxt_joint_job_t *job, nxt_socket_conf_joint_t *joint); + /* The request deadline handler, for the app-timeout test. */ void nxt_router_test_app_timeout(nxt_task_t *task, void *obj, void *data); diff --git a/src/test/nxt_listen_event_test.c b/src/test/nxt_listen_event_test.c new file mode 100644 index 000000000..3e15f9459 --- /dev/null +++ b/src/test/nxt_listen_event_test.c @@ -0,0 +1,321 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * Regression test for M-6: nxt_listen_event() (nxt_conn_accept.c) used to + * return a non-NULL lev even when the spare conn could not be allocated, + * and that lev was neither armed (enable_accept) nor linked into + * engine->listen_connections. Every caller (nxt_router.c, nxt_controller.c, + * nxt_runtime.c) only checks for NULL, so the listener silently never + * accepted on that engine, and the router could not even find it again for + * a later update or close. + * + * The fix keeps the listener: lev is linked, left unarmed, and handed to the + * same recovery the accept path already uses when a spare cannot be + * allocated -- nxt_conn_accept_close_idle(), which closes idle conns and + * arms the 100ms listen timer. nxt_conn_listen_timer_handler() then + * retries the spare and, once it succeeds, arms accept. + * + * Both halves are driven here on a fixture engine: + * + * 1. with engine->connections == engine->max_connections the spare + * allocation fails; nxt_listen_event() must still return a lev that is + * linked, unarmed, without a spare, with the retry timer pending; + * + * 2. a slot is freed (max_connections raised), the listen timer handler + * installed on lev->timer is run; it must allocate the spare, arm + * accept once and invoke the accept handler with that spare. + */ + +#include +#include +#include +#include "nxt_tests.h" + + +static nxt_uint_t nxt_listen_event_test_enable_accept_calls; +static nxt_uint_t nxt_listen_event_test_accept_calls; +static nxt_conn_t *nxt_listen_event_test_accepted_spare; +static uint32_t nxt_listen_event_test_accepted_ready; + + +static void +nxt_listen_event_test_enable_accept(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + nxt_listen_event_test_enable_accept_calls++; + + /* What a real enable_accept does: the read side becomes active. */ + ev->read = NXT_EVENT_ACTIVE; +} + + +static void +nxt_listen_event_test_disable_read(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + /* + * Must not be reached: the listener is never armed before the retry + * succeeds, and the M-5 guard in nxt_conn_accept_close_idle() skips an + * inactive listener. Counted through the enable_accept counter going + * negative would be obscure, so use a separate alert instead. + */ + nxt_thread_log_alert("listen event test: disable_read() on a listener " + "that was never armed"); + + ev->read = NXT_EVENT_INACTIVE; +} + + +static void +nxt_listen_event_test_accept(nxt_task_t *task, void *obj, void *data) +{ + nxt_listen_event_t *lev; + + /* + * Stands in for nxt_conn_io_accept(): the timer handler calls + * lev->accept with the spare it just allocated. Record it instead of + * running accept(2) on the fixture's closed listen fd. + */ + lev = obj; + + nxt_listen_event_test_accept_calls++; + nxt_listen_event_test_accepted_spare = data; + nxt_listen_event_test_accepted_ready = lev->ready; +} + + +static nxt_conn_io_t nxt_listen_event_test_io = { + .accept = nxt_listen_event_test_accept, +}; + + +nxt_int_t +nxt_listen_event_test(nxt_thread_t *thr) +{ + nxt_int_t ret; + nxt_task_t *task; + nxt_sockaddr_t sa; + nxt_listen_event_t *lev; + nxt_listen_socket_t ls; + nxt_event_engine_t engine, *saved_engine; + nxt_work_queue_cache_t cache; + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "listen event test started"); + + ret = NXT_ERROR; + lev = NULL; + + task = thr->task; + task->thread = thr; + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&ls, sizeof(nxt_listen_socket_t)); + nxt_memzero(&sa, sizeof(nxt_sockaddr_t)); + + if (nxt_slow_path(nxt_timers_init(&engine.timers, 4) != NXT_OK)) { + nxt_log_alert(thr->log, "listen event test failed to init timers"); + return NXT_ERROR; + } + + /* Backs the conn structs and the sockaddr cache of the retry below. */ + engine.mem_pool = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(engine.mem_pool == NULL)) { + nxt_free(engine.timers.changes); + return NXT_ERROR; + } + + engine.task = *task; + engine.event.io = &nxt_listen_event_test_io; + engine.event.enable_accept = nxt_listen_event_test_enable_accept; + engine.event.disable_read = nxt_listen_event_test_disable_read; + + nxt_queue_init(&engine.listen_connections); + nxt_queue_init(&engine.idle_connections); + nxt_queue_init(&engine.active_connections); + + nxt_work_queue_cache_create(&cache, 1024); + + engine.fast_work_queue.cache = &cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + + engine.close_work_queue.cache = &cache; + nxt_work_queue_name(&engine.close_work_queue, "close"); + + /* + * Enough of a listen socket for nxt_conn_accept_alloc(): the spare's + * sockaddr is sized and typed from it. The fd is never used, since + * accept is stubbed above. + */ + sa.type = SOCK_STREAM; + sa.u.sockaddr.sa_family = AF_INET; + + ls.socket = -1; + ls.sockaddr = &sa; + ls.socklen = sizeof(struct sockaddr_in); + ls.address_length = NXT_INET_ADDR_STR_LEN; + + /* + * engine->connections < engine->max_connections is false (0 < 0), so + * nxt_conn_accept_alloc() fails immediately without touching memory + * pools or the listen socket's sockaddr cache -- the failure this test + * needs, reached the cheapest way. + */ + engine.connections = 0; + engine.max_connections = 0; + + /* The timer-driven first accept must start a batch of this size. */ + engine.batch = 3; + + saved_engine = thr->engine; + thr->engine = &engine; + + nxt_listen_event_test_enable_accept_calls = 0; + nxt_listen_event_test_accept_calls = 0; + nxt_listen_event_test_accepted_spare = NULL; + + lev = nxt_listen_event(task, &ls); + + if (nxt_slow_path(lev == NULL)) { + nxt_log_alert(thr->log, "listen event test: nxt_listen_event() " + "returned NULL although only the spare conn could " + "not be allocated"); + goto done; + } + + if (nxt_slow_path(nxt_queue_is_empty(&engine.listen_connections) + || nxt_queue_first(&engine.listen_connections) + != &lev->link)) + { + nxt_log_alert(thr->log, "listen event test: the listener was not " + "linked into listen_connections"); + goto done; + } + + if (nxt_slow_path(lev->next != NULL)) { + nxt_log_alert(thr->log, "listen event test: a spare conn is set " + "although its allocation must have failed"); + goto done; + } + + if (nxt_slow_path(nxt_listen_event_test_enable_accept_calls != 0 + || nxt_fd_event_is_active(lev->socket.read))) + { + nxt_log_alert(thr->log, "listen event test: accept was armed " + "without a spare conn"); + goto done; + } + + if (nxt_slow_path(!nxt_timer_is_in_tree(&lev->timer) + && lev->timer.change == NXT_TIMER_NO_CHANGE)) + { + nxt_log_alert(thr->log, "listen event test: the listen timer was " + "not armed, the spare would never be retried"); + goto done; + } + + if (nxt_slow_path(lev->timer.handler == NULL)) { + nxt_log_alert(thr->log, "listen event test: no listen timer " + "handler installed"); + goto done; + } + + /* + * The idle-close pass runs later, so it must have the engine task: the + * task of the caller is the task of a router job, which is freed with + * the configuration after the job is posted. + */ + if (nxt_slow_path(engine.close_work_queue.head == NULL + || engine.close_work_queue.head->task != &engine.task)) + { + nxt_log_alert(thr->log, "listen event test: the idle-close pass " + "was not queued with the engine task"); + goto done; + } + + /* + * nxt_conn_accept_close_idle() also queued the idle-close pass. There + * is nothing idle on the fixture, so it is a no-op here; run it anyway + * so the queue is empty when the cache is destroyed below. + */ + while (engine.close_work_queue.head != NULL) { + void *obj, *data; + nxt_task_t *t; + nxt_work_handler_t handler; + + handler = nxt_work_queue_pop(&engine.close_work_queue, &t, &obj, + &data); + handler(t, obj, data); + } + + /* + * A connection slot frees up. Run what the 100ms timer would run: + * nxt_conn_listen_timer_handler(), installed on lev->timer by + * nxt_listen_event(). It must now get the spare, arm accept and hand + * the spare to the accept handler. + */ + engine.max_connections = 1; + + lev->timer.handler(task, &lev->timer, NULL); + + if (nxt_slow_path(lev->next == NULL)) { + nxt_log_alert(thr->log, "listen event test: the retry did not " + "allocate the spare conn"); + goto done; + } + + if (nxt_slow_path(nxt_listen_event_test_enable_accept_calls != 1 + || !nxt_fd_event_is_active(lev->socket.read))) + { + nxt_log_alert(thr->log, "listen event test: the retry did not arm " + "accept exactly once (%ui calls)", + nxt_listen_event_test_enable_accept_calls); + goto done; + } + + if (nxt_slow_path(nxt_listen_event_test_accept_calls != 1 + || nxt_listen_event_test_accepted_spare != lev->next)) + { + nxt_log_alert(thr->log, "listen event test: the retry did not " + "call the accept handler with the new spare " + "(%ui calls)", nxt_listen_event_test_accept_calls); + goto done; + } + + if (nxt_slow_path(nxt_listen_event_test_accepted_ready != 3)) { + nxt_log_alert(thr->log, "listen event test: the first accept after " + "the retry got ready %uD, expected the batch 3", + nxt_listen_event_test_accepted_ready); + goto done; + } + + ret = NXT_OK; + +done: + + if (lev != NULL) { + /* What nxt_router_listen_event_release() does on the last ref. */ + + if (lev->next != NULL) { + nxt_sockaddr_cache_free(&engine, lev->next); + nxt_conn_free(task, lev->next); + } + + nxt_timer_delete(&engine, &lev->timer); + nxt_queue_remove(&lev->link); + nxt_free(lev); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_mp_destroy(engine.mem_pool); + nxt_free(engine.timers.changes); + + if (nxt_fast_path(ret == NXT_OK)) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "listen event test passed"); + } + + return ret; +} diff --git a/src/test/nxt_router_stale_joint_test.c b/src/test/nxt_router_stale_joint_test.c new file mode 100644 index 000000000..9a16e72e7 --- /dev/null +++ b/src/test/nxt_router_stale_joint_test.c @@ -0,0 +1,547 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * When nxt_listen_event() cannot allocate the listen event, the router + * create job acknowledges the configuration but leaves its joint in + * engine->joints, with the listen socket reference that the job took. A + * later update or delete of that listener must release both. Otherwise + * the old configurations stay referenced, engine->joints never becomes + * empty (so a worker thread that has to quit never exits), and a listen + * socket that no engine could use is never closed. + * + * The stale joints are matched by the listen socket: when an engine is + * removed, the delete job has the socket conf of the new configuration, + * and the stale joint has the old one. + * + * The fixture is a set of joints in one engine. The socket confs and the + * listen sockets keep more than one reference, so a release only unlinks + * the joint and drops one reference of each; the test reads the counts. + * + * nxt_router_stale_joint_jobs() then runs the real create and update jobs + * on a fixture engine: a create whose listen event allocation fails must + * keep its joint and take its listen socket reference, and the next update + * of that listener must release both. + */ + +#include +#include +#include +#include +#include +#include "nxt_tests.h" + + +#define NXT_STALE_JOINTS 5 + + +static nxt_bool_t +nxt_router_stale_joint_linked(nxt_event_engine_t *engine, + nxt_socket_conf_joint_t *joint) +{ + nxt_queue_link_t *qlk; + + for (qlk = nxt_queue_first(&engine->joints); + qlk != nxt_queue_tail(&engine->joints); + qlk = nxt_queue_next(qlk)) + { + if (qlk == &joint->link) { + return 1; + } + } + + return 0; +} + + +static nxt_int_t +nxt_router_stale_joint_check(nxt_thread_t *thr, const char *step, + nxt_event_engine_t *engine, nxt_socket_conf_joint_t *joint, + const nxt_bool_t *linked) +{ + nxt_uint_t i; + + for (i = 0; i < NXT_STALE_JOINTS; i++) { + if (nxt_router_stale_joint_linked(engine, &joint[i]) != linked[i]) { + nxt_log_alert(thr->log, "router stale joint test failed: %s: " + "joint %ui linked %d, expected %d", step, i, + !linked[i], linked[i]); + return NXT_ERROR; + } + + if (joint[i].socket_conf->count != (linked[i] ? 2 : 1)) { + nxt_log_alert(thr->log, "router stale joint test failed: %s: " + "socket conf %ui count %D", step, i, + joint[i].socket_conf->count); + return NXT_ERROR; + } + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_stale_joint_last(nxt_thread_t *thr, nxt_router_conf_t *rtcf, + nxt_event_engine_t *engine) +{ + int fd; + nxt_sockaddr_t sa; + nxt_socket_conf_t skcf; + nxt_listen_socket_t *ls; + nxt_socket_conf_joint_t joint; + + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd == -1) { + nxt_log_alert(thr->log, "router stale joint test: socket() failed %E", + nxt_errno); + return NXT_ERROR; + } + + ls = nxt_zalloc(sizeof(nxt_listen_socket_t)); + if (ls == NULL) { + (void) close(fd); + return NXT_ERROR; + } + + nxt_memzero(&sa, sizeof(nxt_sockaddr_t)); + sa.u.sockaddr.sa_family = AF_INET; + + ls->socket = fd; + ls->sockaddr = &sa; + ls->count = 1; + + nxt_memzero(&skcf, sizeof(nxt_socket_conf_t)); + skcf.count = 2; + skcf.router_conf = rtcf; + skcf.listen = ls; + + nxt_memzero(&joint, sizeof(nxt_socket_conf_joint_t)); + joint.count = 1; + joint.stale = 1; + joint.socket_conf = &skcf; + joint.engine = engine; + + nxt_queue_insert_tail(&engine->joints, &joint.link); + + /* ls is freed here; only the saved fd is used after the call. */ + nxt_router_test_listen_socket_release_stale(thr->task, engine, ls, NULL); + + if (nxt_router_stale_joint_linked(engine, &joint)) { + nxt_log_alert(thr->log, "router stale joint test failed: last: " + "the joint is still linked"); + return NXT_ERROR; + } + + if (fcntl(fd, F_GETFD) != -1 || errno != EBADF) { + nxt_log_alert(thr->log, "router stale joint test failed: last: " + "the listen socket %d was not closed", fd); + (void) close(fd); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_uint_t nxt_router_stale_joint_signals; + + +static void +nxt_router_stale_joint_signal(nxt_event_engine_t *engine, nxt_uint_t signo) +{ + nxt_router_stale_joint_signals++; +} + + +static void +nxt_router_stale_joint_enable_accept(nxt_event_engine_t *engine, + nxt_fd_event_t *ev) +{ + ev->read = NXT_EVENT_ACTIVE; +} + + +static void +nxt_router_stale_joint_accept(nxt_task_t *task, void *obj, void *data) +{ +} + + +static nxt_conn_io_t nxt_router_stale_joint_io = { + .accept = nxt_router_stale_joint_accept, +}; + + +/* + * Checks that the job was acknowledged: nxt_router_conf_wait_post() posts + * the work of the job to the engine of the temporary configuration. + */ + +static nxt_int_t +nxt_router_stale_joint_posted(nxt_thread_t *thr, const char *step, + nxt_event_engine_t *conf_engine, nxt_joint_job_t *job) +{ + if (conf_engine->locked_work_queue.head != &job->work + || nxt_router_stale_joint_signals != 1) + { + nxt_log_alert(thr->log, "router stale joint test failed: %s: the " + "job was not acknowledged", step); + return NXT_ERROR; + } + + conf_engine->locked_work_queue.head = NULL; + conf_engine->locked_work_queue.tail = NULL; + nxt_router_stale_joint_signals = 0; + + return NXT_OK; +} + + +static nxt_int_t +nxt_router_stale_joint_jobs(nxt_thread_t *thr, nxt_router_conf_t *rtcf) +{ + void *obj, *data; + nxt_int_t ret; + nxt_uint_t i; + nxt_task_t *task, *t; + nxt_sockaddr_t sa; + nxt_joint_job_t job[2]; + nxt_queue_link_t *qlk; + nxt_socket_conf_t skcf[2]; + nxt_event_engine_t engine, conf_engine, *saved_engine; + nxt_listen_event_t *lev; + nxt_work_handler_t handler; + nxt_listen_socket_t ls; + nxt_work_queue_cache_t cache; + nxt_router_temp_conf_t tmcf; + nxt_socket_conf_joint_t joint[2]; + + ret = NXT_ERROR; + lev = NULL; + task = thr->task; + + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&conf_engine, sizeof(nxt_event_engine_t)); + nxt_memzero(&tmcf, sizeof(nxt_router_temp_conf_t)); + nxt_memzero(job, sizeof(job)); + nxt_memzero(skcf, sizeof(skcf)); + nxt_memzero(joint, sizeof(joint)); + nxt_memzero(&ls, sizeof(nxt_listen_socket_t)); + nxt_memzero(&sa, sizeof(nxt_sockaddr_t)); + + if (nxt_timers_init(&engine.timers, 4) != NXT_OK) { + return NXT_ERROR; + } + + engine.mem_pool = nxt_mp_create(1024, 128, 256, 32); + if (engine.mem_pool == NULL) { + nxt_free(engine.timers.changes); + return NXT_ERROR; + } + + engine.task = *task; + engine.event.io = &nxt_router_stale_joint_io; + engine.event.enable_accept = nxt_router_stale_joint_enable_accept; + + nxt_queue_init(&engine.joints); + nxt_queue_init(&engine.listen_connections); + nxt_queue_init(&engine.idle_connections); + + nxt_work_queue_cache_create(&cache, 0); + + engine.fast_work_queue.cache = &cache; + engine.close_work_queue.cache = &cache; + + /* The engine that gets the acknowledgements of the jobs. */ + conf_engine.task = *task; + conf_engine.event.signal = nxt_router_stale_joint_signal; + tmcf.engine = &conf_engine; + + /* + * The engine is at max_connections, so the listen event gets no spare + * conn: it stays unarmed, and its listen timer retries the spare. The + * listen socket is not used for I/O. + */ + sa.type = SOCK_STREAM; + sa.u.sockaddr.sa_family = AF_INET; + + ls.socket = -1; + ls.sockaddr = &sa; + ls.socklen = sizeof(struct sockaddr_in); + ls.count = 10; + + engine.max_connections = 0; + + for (i = 0; i < 2; i++) { + job[i].task = *task; + job[i].tmcf = &tmcf; + + skcf[i].count = 2; + skcf[i].router_conf = rtcf; + skcf[i].listen = &ls; + + joint[i].count = 1; + joint[i].socket_conf = &skcf[i]; + joint[i].engine = &engine; + } + + saved_engine = thr->engine; + thr->engine = &engine; + nxt_router_stale_joint_signals = 0; + + /* + * A create whose listen event cannot be allocated: the joint stays + * linked, the engine holds one listen socket reference, and the job is + * acknowledged. + */ + + nxt_listen_event_test_alloc_failures = 1; + + nxt_router_test_listen_socket_create(task, &job[0], &joint[0]); + + if (!nxt_router_stale_joint_linked(&engine, &joint[0]) + || !nxt_queue_is_empty(&engine.listen_connections)) + { + nxt_log_alert(thr->log, "router stale joint test failed: create: " + "the joint is not linked, or a listen event exists"); + goto done; + } + + if (ls.count != 11 || !joint[0].stale) { + nxt_log_alert(thr->log, "router stale joint test failed: create: " + "listen count %D (expected 11), stale flag %d", + ls.count, joint[0].stale); + goto done; + } + + if (nxt_router_stale_joint_posted(thr, "create", &conf_engine, &job[0]) + != NXT_OK) + { + goto done; + } + + /* + * The update of the same listener finds no listen event, so it creates + * one, and releases the stale joint of the failed create with its + * listen socket reference. + */ + + nxt_router_test_listen_socket_update(task, &job[1], &joint[1]); + + if (nxt_queue_is_empty(&engine.listen_connections)) { + nxt_log_alert(thr->log, "router stale joint test failed: update: " + "no listen event was created"); + goto done; + } + + qlk = nxt_queue_first(&engine.listen_connections); + lev = nxt_queue_link_data(qlk, nxt_listen_event_t, link); + + if (nxt_router_stale_joint_linked(&engine, &joint[0]) + || !nxt_router_stale_joint_linked(&engine, &joint[1])) + { + nxt_log_alert(thr->log, "router stale joint test failed: update: " + "the stale joint is still linked, or the new one is " + "not"); + goto done; + } + + if (ls.count != 11 || skcf[0].count != 1 || skcf[1].count != 2 + || joint[1].stale) + { + nxt_log_alert(thr->log, "router stale joint test failed: update: " + "listen count %D (expected 11), socket conf counts " + "%D %D (expected 1 2)", ls.count, skcf[0].count, + skcf[1].count); + goto done; + } + + if (nxt_router_stale_joint_posted(thr, "update", &conf_engine, &job[1]) + != NXT_OK) + { + goto done; + } + + ret = NXT_OK; + +done: + + nxt_listen_event_test_alloc_failures = 0; + + if (lev != NULL) { + nxt_timer_delete(&engine, &lev->timer); + nxt_queue_remove(&lev->link); + nxt_free(lev); + } + + /* The idle-close pass that the missing spare conn queued. */ + while (engine.close_work_queue.head != NULL) { + handler = nxt_work_queue_pop(&engine.close_work_queue, &t, &obj, + &data); + handler(t, obj, data); + } + + thr->engine = saved_engine; + + nxt_work_queue_cache_destroy(&cache); + nxt_mp_destroy(engine.mem_pool); + nxt_free(engine.timers.changes); + + return ret; +} + + +nxt_int_t +nxt_router_stale_joint_test(nxt_thread_t *thr) +{ + nxt_uint_t i; + nxt_router_t router; + nxt_router_conf_t rtcf; + nxt_event_engine_t engine; + nxt_listen_socket_t ls[3]; + nxt_socket_conf_t skcf[NXT_STALE_JOINTS]; + nxt_socket_conf_joint_t joint[NXT_STALE_JOINTS]; + + static const nxt_uint_t listen_of[] = { 0, 1, 0, 1, 1 }; + static const nxt_bool_t after_update[] = { 0, 1, 1, 1, 1 }; + static const nxt_bool_t after_delete[] = { 0, 0, 1, 0, 0 }; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router stale joint test started"); + + nxt_memzero(&router, sizeof(nxt_router_t)); + nxt_memzero(&rtcf, sizeof(nxt_router_conf_t)); + nxt_memzero(&engine, sizeof(nxt_event_engine_t)); + nxt_memzero(ls, sizeof(ls)); + nxt_memzero(skcf, sizeof(skcf)); + nxt_memzero(joint, sizeof(joint)); + + rtcf.router = &router; + rtcf.count = 100; + + nxt_queue_init(&engine.joints); + + for (i = 0; i < nxt_nitems(ls); i++) { + ls[i].socket = -1; + ls[i].count = 10; + } + + /* + * 0: the stale joint of a failed create of listener A. + * 1: the stale joint of a failed create of listener B. + * 2: the new joint of the update of listener A (the job's own joint). + * 3, 4: more stale joints of listener B, each with its own socket conf, + * as after failed creates of B in older configurations. + */ + + for (i = 0; i < NXT_STALE_JOINTS; i++) { + skcf[i].count = 2; + skcf[i].router_conf = &rtcf; + skcf[i].listen = &ls[listen_of[i]]; + + joint[i].count = 1; + joint[i].stale = (i != 2); + joint[i].socket_conf = &skcf[i]; + joint[i].engine = &engine; + + nxt_queue_insert_tail(&engine.joints, &joint[i].link); + } + + /* + * An update of listener A keeps its own joint and releases only the + * stale joint 0 and its reference on A. + */ + + nxt_router_test_listen_socket_release_stale(thr->task, &engine, &ls[0], + &joint[2]); + + if (nxt_router_stale_joint_check(thr, "update", &engine, joint, + after_update) + != NXT_OK) + { + return NXT_ERROR; + } + + if (ls[0].count != 9 || ls[1].count != 10) { + nxt_log_alert(thr->log, "router stale joint test failed: update: " + "listen counts %D %D, expected 9 10", + ls[0].count, ls[1].count); + return NXT_ERROR; + } + + /* + * A delete of listener B releases all its joints, whatever socket + * conf they have, and one reference on B for each. + */ + + nxt_router_test_listen_socket_release_stale(thr->task, &engine, &ls[1], + NULL); + + if (nxt_router_stale_joint_check(thr, "delete", &engine, joint, + after_delete) + != NXT_OK) + { + return NXT_ERROR; + } + + if (ls[0].count != 9 || ls[1].count != 7) { + nxt_log_alert(thr->log, "router stale joint test failed: delete: " + "listen counts %D %D, expected 9 7", + ls[0].count, ls[1].count); + return NXT_ERROR; + } + + /* + * A joint without the stale flag stays, also when no joint is kept: + * it can be the joint of a closed listener that a request still holds. + */ + + nxt_router_test_listen_socket_release_stale(thr->task, &engine, &ls[0], + NULL); + + if (nxt_router_stale_joint_check(thr, "not stale", &engine, joint, + after_delete) + != NXT_OK) + { + return NXT_ERROR; + } + + /* A listen socket without stale joints: nothing changes. */ + + nxt_router_test_listen_socket_release_stale(thr->task, &engine, &ls[2], + NULL); + + if (nxt_router_stale_joint_check(thr, "no match", &engine, joint, + after_delete) + != NXT_OK) + { + return NXT_ERROR; + } + + if (ls[2].count != 10 || rtcf.count != 100) { + nxt_log_alert(thr->log, "router stale joint test failed: no match: " + "listen count %D, router conf count %D", + ls[2].count, rtcf.count); + return NXT_ERROR; + } + + /* + * The last reference closes the listen socket and frees it. This is + * the case of a listener whose listen event could not be allocated on + * any engine: before, nothing closed it, and the address stayed bound. + */ + + if (nxt_router_stale_joint_last(thr, &rtcf, &engine) != NXT_OK) { + return NXT_ERROR; + } + + if (nxt_router_stale_joint_jobs(thr, &rtcf) != NXT_OK) { + return NXT_ERROR; + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "router stale joint test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 8a9700733..29b7b8950 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -286,6 +286,10 @@ main(int argc, char **argv) return 1; } + if (nxt_listen_event_test(thr) != NXT_OK) { + return 1; + } + if (nxt_conn_io_accept_test(thr) != NXT_OK) { return 1; } @@ -338,6 +342,10 @@ main(int argc, char **argv) return 1; } + if (nxt_router_stale_joint_test(thr) != NXT_OK) { + return 1; + } + if (nxt_router_websocket_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index c54d7260b..6b6559c46 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -81,6 +81,7 @@ nxt_int_t nxt_port_mmap_read_test(nxt_thread_t *thr); nxt_int_t nxt_port_mmaps_max_test(nxt_thread_t *thr); nxt_int_t nxt_port_ready_test(nxt_thread_t *thr); nxt_int_t nxt_conn_close_idle_test(nxt_thread_t *thr); +nxt_int_t nxt_listen_event_test(nxt_thread_t *thr); nxt_int_t nxt_conn_io_accept_test(nxt_thread_t *thr); nxt_int_t nxt_runtime_idle_close_test(nxt_thread_t *thr); nxt_int_t nxt_router_new_port_test(nxt_thread_t *thr); @@ -94,6 +95,7 @@ nxt_int_t nxt_router_app_timeout_test(nxt_thread_t *thr); nxt_int_t nxt_router_remove_pid_soak_test(nxt_thread_t *thr); nxt_int_t nxt_router_detached_test(nxt_thread_t *thr); nxt_int_t nxt_router_sender_test(nxt_thread_t *thr); +nxt_int_t nxt_router_stale_joint_test(nxt_thread_t *thr); nxt_int_t nxt_router_websocket_test(nxt_thread_t *thr); nxt_int_t nxt_router_prepare_msg_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); From b12fb67788838f9ffe829df98adbe4c7a523cf6d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 08:29:04 +0200 Subject: [PATCH 331/391] main: store the state files in a short-lived child process Since commit 9c0ab65c73f29770c82cd98b115e011cd867f865, each store of conf.json runs two fsync(2) calls in the event loop of the main process. The first store of a run also writes the version file, so it runs four. On pro4s-4 (ext4 on an md RAID of SATA SSDs) each call costs about 11 ms. A reconfiguration then took 42.2 ms instead of 19.1 ms to its first 200 from the new application, and SIGTERM took 17.4 ms instead of 5.0 ms to the exit of main. While main waited, it forked no worker and handled no signal. See https://github.com/freeunitorg/freeunit/issues/516. nxt_main_port_conf_store_handler() still checks the sender, the fd and the size, and maps the configuration. It then gives the mapping to a scheduler. The scheduler forks a child. The child runs the version store and the conf.json store with nxt_main_file_store(), as main did, with the same alerts. It exits with _exit(0) on success and _exit(1) on failure, so it runs no atexit() handler of main. It does not use the event engine, the ports or the process registry. It writes only the state files and the log. Main unmaps its copy of the mapping and goes back to its loop. Only one store child runs at a time. Two children would race on the same temporary name. A store that arrives while a child runs becomes the pending store. A newer store replaces it and unmaps the older mapping. The SIGCHLD handler recognises the store child before the process registry lookup and starts the pending store. The store child is not in the registry and is not counted in rt->nprocesses. The child keeps the signal mask of main, so the signals of main stay blocked. So SIGTERM or SIGINT sent to the whole process group, for example by "systemctl stop" or Ctrl-C, does not stop a store. Before this change, main also finished a store before it handled such a signal. SIGKILL still stops the child. When the store child exits with a non-zero code or on a signal, main logs "state store child N failed, the configuration was not stored". The child logs its own alerts too, but after SIGUSR1 they can go to the rotated log, because only main reopens the log. At exit, main waits for the store child and then for the pending store, in its event loop. The SIGCHLD handler quits main only when the last registered process and the store child are gone. Main does not wait with a blocking call. Main does not kill the store child: nxt_runtime_stop_all_processes() and nxt_runtime_exit() act only on processes in the registry. Main waits because in a container it is PID 1 (pkg/docker/docker-entrypoint.sh ends with exec "$@"), and the kernel kills every other process of the PID namespace when PID 1 exits. A store child that outlived main would then die, and the configuration that got 200 would be lost. The wait adds time to the exit only when a store is still running. nxt_conf_ver is set in main only when a child that stored the version file exits with 0. Until then each store writes the version file again. This is what main did before when a version store failed. When fork() fails, main logs "fork() failed for the state store" and does the store itself, as before, so a store is not lost. The runtime thread pool was not used. nxt_runtime_thread_pools() creates one pool in main, and nothing posts to it, so it never starts. A store in that pool would make main multi-threaded while it forks workers. A forked worker inherits rt->thread_pools with the pool marked ready but without its threads, and nxt_thread_pool_destroy() in that worker would post an exit work item that no thread runs. The fork does not touch any of this. Tests: three new cases in test/test_state_store.py. A PUT and then SIGTERM, and four PUTs and then SIGTERM, must leave the last configuration in conf.json, with no process of the group left when main has exited, and a restart must load it. Six PUTs in quick succession must leave the last configuration, with no temporary file and no alert. In a --debug build, this case also checks in the log that no store child starts before the previous one exits. The existing ENOSPC case now also expects the alert of main. These tests need --restart, so build-test.yml and the python leg of sanitize.yml run them in their --restart steps. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 9 +- .github/workflows/sanitize.yml | 5 + CHANGES | 8 + docs/changes.xml | 11 ++ src/nxt_main_process.c | 255 ++++++++++++++++++++++++++----- test/test_state_store.py | 191 ++++++++++++++++++++++- 6 files changed, 438 insertions(+), 41 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d4f3e4a08..5dc4a0cdc 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -960,14 +960,17 @@ jobs: if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} # test_asgi_lifespan.py stops unitd from inside the test to check the - # shutdown event, and test_tls_certificate_update_restart restarts it - # to check that a replaced bundle survives. Both need --restart, which - # the run above does not pass, so they skip there. Run them again. + # shutdown event, test_tls_certificate_update_restart restarts it to + # check that a replaced bundle survives, and test_state_store.py runs + # unitd on its own state directory and stops it while a store runs. + # All need --restart, which the run above does not pass, so they skip + # there. Run them again. - name: Run the tests that restart unitd, with --restart run: | export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" sudo -E /usr/bin/pytest-3 --print-log --restart \ test/test_asgi_lifespan.py \ + test/test_state_store.py \ test/test_tls.py::test_tls_certificate_update_restart if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 7569b165a..dd89fea73 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -139,8 +139,13 @@ jobs: # (#276), and skips on any build without --debug. Every # build-test leg is a release build, so this leg is the only one # in which that test runs rather than skips. + # + # test_state_store forks a store child from main and stops main + # while the child runs (#516); it needs --restart, and this is + # the leg that runs that path under ASan. tests: >- test/test_graceful_reload.py + test/test_state_store.py test/test_listener_drain.py test/test_static.py test/test_conn_task_race.py diff --git a/CHANGES b/CHANGES index 88568671d..e70344639 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Change: the main process now writes the state files in a + short-lived child process, so a slow fsync(2) no longer delays + the start of application processes after a reconfiguration, or + the exit of the main process. One store runs at a time; a newer + configuration waits for it, and only the latest one waiting is + stored. At exit, the main process waits for the store to finish. + When the store fails, the main process logs an alert. + *) Security: the router and libunit read records from shared memory that the other process can still write, without enough checks. A buffer that was not a whole number of records was read past its diff --git a/docs/changes.xml b/docs/changes.xml index b2bb20650..06d7b6a29 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,17 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +the main process now writes the state files in a short-lived child +process, so a slow fsync(2) no longer delays the start of application +processes after a reconfiguration, or the exit of the main process. One +store runs at a time; a newer configuration waits for it, and only the +latest one waiting is stored. At exit, the main process waits for the +store to finish. When the store fails, the main process logs an alert. + + + when unitd ran as root and the control socket was given to another user or diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index cda28c8e2..51aed83c1 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -35,6 +35,15 @@ typedef struct { } nxt_conf_app_map_t; +/* The state store runs in a short-lived child of main, one at a time. */ +typedef struct { + nxt_pid_t pid; /* The running store child, or 0. */ + nxt_bool_t version; /* It also stores the version file. */ + u_char *pending; /* The next store's mapping, or NULL. */ + size_t pending_size; +} nxt_main_store_t; + + static nxt_int_t nxt_main_process_port_create(nxt_task_t *task, nxt_runtime_t *rt); static void nxt_main_process_title(nxt_task_t *task); @@ -68,6 +77,13 @@ static void nxt_main_remove_child_pid_handler(nxt_task_t *task, #endif static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +static void nxt_main_store_schedule(nxt_task_t *task, u_char *p, + size_t size); +static void nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size); +static nxt_int_t nxt_main_store_files(nxt_task_t *task, u_char *p, + size_t size, nxt_bool_t version); +static nxt_bool_t nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, + int status); static nxt_int_t nxt_main_file_store_inherit(nxt_task_t *task, nxt_file_t *tmp, const char *name); static void nxt_main_port_access_log_handler(nxt_task_t *task, @@ -124,6 +140,8 @@ nxt_uint_t nxt_conf_ver; static nxt_bool_t nxt_exiting; +static nxt_main_store_t nxt_main_store; + nxt_int_t nxt_main_process_start(nxt_thread_t *thr, nxt_task_t *task, @@ -1444,9 +1462,13 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) * for the whole handler. */ - process = nxt_runtime_process_find(rt, pid); + if (!nxt_main_store_exited(task, pid, status)) { + process = nxt_runtime_process_find(rt, pid); + + if (process == NULL) { + continue; + } - if (process != NULL) { nxt_main_process_cleanup(task, process); /* @@ -1483,35 +1505,46 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) nxt_process_close_ports(task, child); } nxt_queue_loop; - if (rt->nprocesses <= 1) { - nxt_runtime_quit(task, 0); - - return; - } + } else { + nxt_port_remove_notify_others(task, process); - continue; - } + nxt_queue_each(child, &children, nxt_process_t, link) { + nxt_port_remove_notify_others(task, child); - nxt_port_remove_notify_others(task, process); + nxt_process_unlink(child); - nxt_queue_each(child, &children, nxt_process_t, link) { - nxt_port_remove_notify_others(task, child); + nxt_process_close_ports(task, child); + } nxt_queue_loop; - nxt_process_unlink(child); + init = *(nxt_process_init_t *) nxt_process_init(process); - nxt_process_close_ports(task, child); - } nxt_queue_loop; + nxt_process_close_ports(task, process); - init = *(nxt_process_init_t *) nxt_process_init(process); + if (init.restart) { + ret = nxt_process_init_start(task, init); + if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_alert(task, "failed to restart %s", init.name); + } + } - nxt_process_close_ports(task, process); + continue; + } + } - if (init.restart) { - ret = nxt_process_init_start(task, init); - if (nxt_slow_path(ret == NXT_ERROR)) { - nxt_alert(task, "failed to restart %s", init.name); - } + /* + * At exit, main also waits for the store child. In a container + * main is PID 1, and when it exits the kernel kills the child. + */ + if (nxt_exiting && rt->nprocesses <= 1) { + if (nxt_main_store.pid != 0) { + nxt_debug(task, "waiting for state store child %PI", + nxt_main_store.pid); + continue; } + + nxt_runtime_quit(task, 0); + + return; } } } @@ -2090,11 +2123,9 @@ static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { void *p; - size_t n, size; - nxt_int_t ret; + size_t size; nxt_port_t *ctl_port; nxt_runtime_t *rt; - u_char ver[NXT_INT_T_LEN]; rt = task->thread->runtime; @@ -2149,37 +2180,187 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "conf_store_handler(%uz): %*s", size, size, p); - if (nxt_conf_ver != NXT_VERNUM) { + /* The scheduler owns the mapping from here on. */ + nxt_main_store_schedule(task, p, size); + + return; + +error: + + nxt_alert(task, "failed to store current configuration"); + + if (p != MAP_FAILED) { + nxt_mem_munmap(p, size); + } + + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; + } +} + + +/* + * A store runs fsync(2) twice for each file, which can take tens of + * milliseconds. A short-lived child does it, so main forks workers and + * handles signals in the meantime (issue #516). + * + * Only one store child runs at a time: two would race on the same + * temporary name. A store that arrives while a child runs becomes the + * pending store, and a newer store replaces it. nxt_main_store_exited() + * starts the pending store when the child exits. + * + * "p" is a mapping of "size" bytes; this function takes ownership of it. + */ +static void +nxt_main_store_schedule(nxt_task_t *task, u_char *p, size_t size) +{ + if (nxt_main_store.pid == 0) { + nxt_main_store_start(task, p, size); + return; + } + + if (nxt_main_store.pending != NULL) { + nxt_debug(task, "state store: a newer store replaces the pending one"); + + nxt_mem_munmap(nxt_main_store.pending, nxt_main_store.pending_size); + } + + nxt_main_store.pending = p; + nxt_main_store.pending_size = size; +} + + +static void +nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) +{ + nxt_pid_t pid; + nxt_bool_t version; + + /* + * The version file is stored again with each store until a child that + * stored it exits with 0. Only then does main set nxt_conf_ver. + */ + version = (nxt_conf_ver != NXT_VERNUM); + + pid = fork(); + + if (nxt_slow_path(pid < 0)) { + nxt_alert(task, "fork() failed for the state store %E", nxt_errno); + + /* Do not lose the store: do it in main, as before the child. */ + if (nxt_main_store_files(task, p, size, version) == NXT_OK + && version) + { + nxt_conf_ver = NXT_VERNUM; + } + + nxt_mem_munmap(p, size); + return; + } + + if (pid == 0) { + /* + * The signals of main stay blocked, so SIGTERM or SIGINT sent to + * the process group does not stop the store. _exit() runs no + * atexit() handler of main. + */ + nxt_pid = getpid(); + task->thread->tid = 0; + + if (nxt_main_store_files(task, p, size, version) != NXT_OK) { + _exit(1); + } + + _exit(0); + } + + nxt_debug(task, "state store child %PI", pid); + + /* The child has its own copy of the mapping. */ + nxt_mem_munmap(p, size); + + nxt_main_store.pid = pid; + nxt_main_store.version = version; +} + + +static nxt_int_t +nxt_main_store_files(nxt_task_t *task, u_char *p, size_t size, + nxt_bool_t version) +{ + size_t n; + nxt_int_t ret; + nxt_runtime_t *rt; + u_char ver[NXT_INT_T_LEN]; + + rt = task->thread->runtime; + + if (version) { n = nxt_sprintf(ver, ver + NXT_INT_T_LEN, "%d", NXT_VERNUM) - ver; ret = nxt_main_file_store(task, rt->state, rt->ver_tmp, rt->ver, ver, n); if (nxt_slow_path(ret != NXT_OK)) { - goto error; + goto fail; } - - nxt_conf_ver = NXT_VERNUM; } ret = nxt_main_file_store(task, rt->state, rt->conf_tmp, rt->conf, p, size); if (nxt_fast_path(ret == NXT_OK)) { - goto cleanup; + return NXT_OK; } -error: +fail: nxt_alert(task, "failed to store current configuration"); -cleanup: + return NXT_ERROR; +} - if (p != MAP_FAILED) { - nxt_mem_munmap(p, size); + +/* + * Called by the SIGCHLD handler for every reaped pid. Returns 1 when "pid" + * was the store child. The child is not in the process registry and is not + * counted in rt->nprocesses. + * + * Main logs a failed store again: the alerts of the child can go to a + * rotated log, because SIGUSR1 reopens the log in main only. + */ +static nxt_bool_t +nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) +{ + u_char *p; + size_t size; + + if (pid != nxt_main_store.pid) { + return 0; } - if (msg->fd[0] != -1) { - nxt_fd_close(msg->fd[0]); - msg->fd[0] = -1; + nxt_main_store.pid = 0; + + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { + if (nxt_main_store.version) { + nxt_conf_ver = NXT_VERNUM; + } + + } else { + nxt_alert(task, "state store child %PI failed, " + "the configuration was not stored", pid); } + + p = nxt_main_store.pending; + + if (p != NULL) { + size = nxt_main_store.pending_size; + + nxt_main_store.pending = NULL; + nxt_main_store.pending_size = 0; + + nxt_main_store_start(task, p, size); + } + + return 1; } diff --git a/test/test_state_store.py b/test/test_state_store.py index 1fa771903..b7a6c771d 100644 --- a/test/test_state_store.py +++ b/test/test_state_store.py @@ -10,11 +10,21 @@ The failure is arranged by putting the state directory on a 64 KiB tmpfs, which needs root and CAP_SYS_ADMIN (pytest as root in a --privileged container) as well as --restart; the test skips otherwise. + +Main runs each store in a short-lived child process, one at a time, and waits +for it before it exits (https://github.com/freeunitorg/freeunit/issues/516). +The other tests here check that a configuration answered with 200 is on disk +after an immediate SIGTERM, and that stores in quick succession leave the +last configuration. They need --restart only. They catch a regression only +when a store takes longer than the few milliseconds main needs to exit, so on +a tmpfs they pass either way. """ import json import os +import re import shutil +import signal import subprocess import tempfile import time @@ -85,10 +95,11 @@ def test_state_store_full_filesystem(requires_restart, skip_alert): if mounted.returncode != 0: pytest.skip(f'could not mount a tmpfs: {mounted.stderr.decode()}') - # The two alerts the failed store is expected to log. + # The two alerts of the failed store, and the alert of main about it. skip_alert( r'failed to store current configuration', r'write\(.*conf\.json\.tmp', + r'state store child \d+ failed', ) try: @@ -139,3 +150,181 @@ def test_state_store_full_filesystem(requires_restart, skip_alert): # A failed umount would make rmdir() raise out of the finally and # mask whichever assertion above actually failed. shutil.rmtree(statedir, ignore_errors=True) + + +def numbered_conf(i): + """SMALL_CONF, made different per store by a setting.""" + + return { + "settings": {"http": {"header_read_timeout": 30 + i}}, + **SMALL_CONF, + } + + +def stored_timeout(statedir): + """The setting in conf.json, or None when there is no conf.json.""" + + try: + text = (statedir / 'conf.json').read_text(encoding='utf-8') + except FileNotFoundError: + return None + + return json.loads(text)['settings']['http']['header_read_timeout'] + + +def group_alive(pgid): + try: + os.killpg(pgid, 0) + except ProcessLookupError: + return False + + return True + + +def put_and_terminate(statedir, count): + """Start unitd on "statedir", PUT "count" configurations, and send + SIGTERM to main as soon as the last one is answered. Return the + temporary directory of that unitd once main has exited.""" + + unit = unit_run(state_dir=str(statedir)) + temp_dir = unit['temp_dir'] + + for i in range(count): + assert 'success' in client.conf(numbered_conf(i)), f'PUT {i}' + + os.kill(int(unit['pid']), signal.SIGTERM) + + assert unit['process'].wait(15) == 0, 'main exited with 0' + + # Main exits only after its store child: no process of its group is + # left to finish the store. In a container main is PID 1, and anything + # left would be killed with it. + assert not group_alive(unit['pgid']), 'no store child outlives main' + + log = Path(temp_dir, 'unit.log').read_text(encoding='utf-8') + + unit_stop() + + # The teardown checks only the log of the last unitd; check this one. + Log.check_alerts(log=log) + + return temp_dir + + +def restart_and_get_timeout(statedir): + unit_run(state_dir=str(statedir)) + + return client.conf_get('settings/http/header_read_timeout') + + +def test_state_store_survives_exit(requires_restart): + """A configuration answered with 200 is stored even when SIGTERM + reaches main while the store runs.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + try: + temp_dir = put_and_terminate(statedir, 1) + + stored = stored_timeout(statedir) + loaded = restart_and_get_timeout(statedir) + + shutil.rmtree(temp_dir, ignore_errors=True) + + assert stored == 30, 'stored before exit' + assert loaded == 30, 'loaded on restart' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +def test_state_store_pending_at_exit(requires_restart): + """Stores that wait behind a running store child are not dropped at + exit: the last configuration is on disk.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + try: + temp_dir = put_and_terminate(statedir, 4) + + stored = stored_timeout(statedir) + loaded = restart_and_get_timeout(statedir) + + shutil.rmtree(temp_dir, ignore_errors=True) + + assert stored == 33, 'the last store won' + assert loaded == 33, 'loaded on restart' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +def test_state_store_serialised(requires_restart): + """Stores in quick succession run one after another and leave the last + configuration, with no temporary file and no alert.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + try: + unit_run(state_dir=str(statedir)) + + for i in range(6): + assert 'success' in client.conf(numbered_conf(i)), f'PUT {i}' + + last = numbered_conf(5) + + for _ in range(100): + try: + if stored_timeout(statedir) == 35: + break + except (ValueError, KeyError): + pass + + time.sleep(0.1) + + # A --debug build logs the start of each store child. A store + # child must not start before the previous one exits. + running = None + + events = re.findall( + r'#\d+ state store child (\d+)$|process (\d+) exited', + Log.read(), + re.M, + ) + + for start, exited in events: + if start: + assert running is None, f'child {start} ran with {running}' + running = start + + elif exited == running: + running = None + + assert stored_timeout(statedir) == 35, 'the last store won' + + # A store that ran after the last one would overwrite it; give any + # such store the time to finish before the second look. + time.sleep(1) + + conf = json.loads((statedir / 'conf.json').read_text(encoding='utf-8')) + + assert conf == last, 'conf.json is the last configuration' + assert [p.name for p in statedir.iterdir() if '.tmp' in p.name] == [] + + # Two stores that run at the same time race on the temporary name, + # and one of them logs a failed rename() or a failed store. + alerts = re.findall(r'.+\[alert\].+', Log.read()) + + assert alerts == [], 'no alert' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) From 69eeda499d90fedba621d100a502ad4e9883bc47 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 3 Oct 2026 12:32:28 +0200 Subject: [PATCH 332/391] php: read the target index before the request ends nxt_php_request_handler() read r->app_target after nxt_php_execute(). nxt_php_execute() ends the request with nxt_unit_request_done(). libunit then releases the request memory, and nxt_unit_mmap_release() fills it with 0xA5. So the module stored a value that was not the index of the request. Usually the value is 0xA5 (165). For a "script" target, the index then did not match the last target, and the module called chdir() on every request. Under strace, 100 requests made 100 chdir() calls before this change and 1 after it. A request could also run in the directory of another target: - With 166 or more targets, the target at index 165 skipped its chdir() after a request for another target. - After fastcgi_finish_request(), the script runs on, and the router can put the next request into the released memory. The worker then stored the index of that request. With two targets and two processes, its next request for that target skipped its chdir() and ran in the directory of the first target, in 10 of 10 runs. Read the index once, before nxt_php_execute(). Target indexes follow the hash order of the target names, not the order in the configuration. So the new test gives each of 166 targets its own directory, and requests each target after another one. On the old code, target "t97" (index 165) returns the directory of "t96". Co-Authored-By: Claude Opus 5.5 --- CHANGES | 8 +++++++ docs/changes.xml | 11 +++++++++ src/nxt_php_sapi.c | 9 ++++--- test/test_php_targets.py | 51 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index e70344639..fe1ecf2b1 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: the PHP module read the target index of a request after the + request was released. libunit fills released request memory with + 0xA5, so the module called chdir(2) on every request to a "script" + target. A request could also run in the directory of another + target. This happened with 166 or more targets, and after + fastcgi_finish_request() when the released memory already held a + request for another target. The bug had appeared in 1.18.0. + *) Change: the main process now writes the state files in a short-lived child process, so a slow fsync(2) no longer delays the start of application processes after a reconfiguration, or diff --git a/docs/changes.xml b/docs/changes.xml index 06d7b6a29..240c7922a 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,17 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +the PHP module read the target index of a request after the request was +released. libunit fills released request memory with 0xA5, so the module +called chdir(2) on every request to a "script" target. A request could also +run in the directory of another target. This happened with 166 or more +targets, and after fastcgi_finish_request() when the released memory already +held a request for another target. The bug had appeared in 1.18.0. + + + the main process now writes the state files in a short-lived child diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 30c43b7a7..19594bc8f 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -1081,12 +1081,14 @@ nxt_php_handle_fs_err(nxt_unit_request_info_t *req) static void nxt_php_request_handler(nxt_unit_request_info_t *req) { + uint8_t app_target; nxt_php_target_t *target; nxt_php_run_ctx_t ctx; nxt_unit_request_t *r; r = req->request; - target = &nxt_php_targets[r->app_target]; + app_target = r->app_target; + target = &nxt_php_targets[app_target]; nxt_memzero(&ctx, sizeof(ctx)); @@ -1103,11 +1105,12 @@ nxt_php_request_handler(nxt_unit_request_info_t *req) ctx.script_dirname = target->script_dirname; ctx.script_name = target->script_name; - ctx.chdir = (r->app_target != nxt_php_last_target); + ctx.chdir = (app_target != nxt_php_last_target); nxt_php_execute(&ctx, r); - nxt_php_last_target = ctx.chdir ? -1 : r->app_target; + /* nxt_php_execute() ended the request; r points to released memory. */ + nxt_php_last_target = ctx.chdir ? -1 : app_target; } diff --git a/test/test_php_targets.py b/test/test_php_targets.py index 0f38fea0a..bbda3e999 100644 --- a/test/test_php_targets.py +++ b/test/test_php_targets.py @@ -1,3 +1,5 @@ +from pathlib import Path + from unit.applications.lang.php import ApplicationPHP from unit.option import option @@ -62,6 +64,55 @@ def test_php_application_targets(): assert client.get(url='/')['body'] == 'index' +def test_php_application_targets_cwd(temp_dir): + # libunit fills a released request with 0xA5 (nxt_unit_mmap_release()). + # The module read the target index after the request ended, so it kept + # 0xA5 (165) as the last target. The target at index 165 then skipped + # chdir() and ran in the directory of the previous target. The index + # follows the hash order of the names, so every target gets its own + # directory, and the test requests each of them. + routes = [] + targets = {} + + for i in range(166): + name = f't{i}' + root = f'{temp_dir}/{name}' + + Path(root).mkdir() + Path(f'{root}/index.php').write_text( + ' Date: Sat, 3 Oct 2026 15:00:51 +0200 Subject: [PATCH 333/391] mmap: poison released buffers only in debug builds libunit fills a released shared memory chunk with 0xA5 in nxt_unit_mmap_release(). The router does the same in nxt_port_mmap_buf_completion(). Release builds now skip both fills. Debug builds keep them. The chunks go back to the free map as before. A response larger than 1 KiB travels in shared memory. The router fills the whole incoming buffer, rounded up to 16 KiB chunks. So each such response costs at least a 16 KiB memset on the router thread. libunit fills the size of each buffer it releases. Measured with PHP 8.5 on a 4-core box. unitd ran on 2 cores, with 1.9 of them busy on average, and 8 cells per workload plus a swapped rerun. For a 1 MiB response, router CPU per request fell from 350 us to 288 us (-17.7%, 95% CI -24.1% to -11.4%). Requests per second rose by 18.4% (95% CI +12.0% to +24.8%). For a 1 MiB POST body, server CPU per request changed by -3.3% (95% CI -12.2% to +5.5%). For a 6-byte and a 64 KiB response, every CI includes zero. A null pair of the same build moved by 2 to 3% on CPU per request for the 6-byte response and by -2.4% (95% CI -8.8% to +4.1%) for the 1 MiB response. A stale byte now stays in a released chunk until the chunk is reused. Only the router and the processes of one application map a segment. Readers stop at the size in the message, not at the end of the chunk. nxt_mp keeps its 0x5A fill in release builds. It fills small pool allocations, not 16 KiB chunks. The fill also made a read after release fail in release CI. test_php_application_targets_cwd fails on the PHP code before 69eeda49 only because of the fill. A release build now keeps the stale index, so there the test passes on the old code: 3 of 3 runs. Two changes keep the check: - test_php_application_targets_cwd_finish_request does not need the fill. fastcgi_finish_request() releases the request for /a while the script keeps running. The next request reuses that memory, and it is still running when the old code reads the index. Signal files order the requests, so the test does not depend on timing. On the old code the test failed in 40 of 40 runs, in release and debug builds, and with 8 busy loops on the same 2 CPUs. With the fix it passed in 85 of 90 runs. Of the other 5, 2 failed before the test ran, because conftest.py could not start unitd under load. The other 3 printed no assertion line. - sanitize.yml runs test_php_targets.py in its --debug PHP leg. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/sanitize.yml | 6 +++ src/nxt_port_memory.c | 4 ++ src/nxt_unit.c | 2 + test/test_php_targets.py | 93 +++++++++++++++++++++++++++++++++- 4 files changed, 104 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index dd89fea73..77702811f 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -161,9 +161,15 @@ jobs: # No --restart: test_php_application.py does not signal the master, # and --restart's teardown rmtree trips on test_php_application_ # forbidden's restricted-permission fixture (rmtree PermissionError). + # + # test_php_targets is here for test_php_application_targets_cwd. + # On the old code it fails only when libunit fills released + # memory, and only a --debug build does that. Every build-test + # leg is a release build. pytest_opts: "" tests: >- test/test_php_application.py + test/test_php_targets.py test/test_app_lifecycle.py # Go is where the app-side port-fd double-close actually manifested # (a flaky "close(N) failed: Bad file descriptor" in the go isolation diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index e680013c1..315e84382 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -157,8 +157,10 @@ nxt_port_mmaps_destroy(nxt_port_mmaps_t *port_mmaps, nxt_bool_t free_elts) } +#if (NXT_DEBUG) #define nxt_port_mmap_free_junk(p, size) \ memset((p), 0xA5, size) +#endif static void @@ -220,7 +222,9 @@ nxt_port_mmap_buf_completion(nxt_task_t *task, void *obj, void *data) c = nxt_port_mmap_chunk_id(hdr, p); } +#if (NXT_DEBUG) nxt_port_mmap_free_junk(p, b->mem.end - p); +#endif nxt_debug(task, "mmap buf completion: %p [%p,%uz] (sent=%d), " "%PI->%PI,%d,%d", b, b->mem.start, b->mem.end - b->mem.start, diff --git a/src/nxt_unit.c b/src/nxt_unit.c index cc2a64b5b..04f44c993 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -5591,7 +5591,9 @@ nxt_unit_mmap_release(nxt_unit_ctx_t *ctx, nxt_port_mmap_header_t *hdr, nxt_chunk_id_t c; nxt_unit_impl_t *lib; +#if (NXT_DEBUG) memset(start, 0xA5, size); +#endif p = start; end = p + size; diff --git a/test/test_php_targets.py b/test/test_php_targets.py index bbda3e999..ec30cf0dc 100644 --- a/test/test_php_targets.py +++ b/test/test_php_targets.py @@ -2,6 +2,7 @@ from unit.applications.lang.php import ApplicationPHP from unit.option import option +from unit.utils import waitforfiles prerequisites = {'modules': {'php': 'any'}} @@ -70,7 +71,9 @@ def test_php_application_targets_cwd(temp_dir): # 0xA5 (165) as the last target. The target at index 165 then skipped # chdir() and ran in the directory of the previous target. The index # follows the hash order of the names, so every target gets its own - # directory, and the test requests each of them. + # directory, and the test requests each of them. libunit fills released + # memory only in --debug builds, so this test fails on the old code only + # there. The test below does not depend on the fill. routes = [] targets = {} @@ -113,6 +116,94 @@ def test_php_application_targets_cwd(temp_dir): assert client.get(url=f'/{name}')['body'] == cwd, name +def test_php_application_targets_cwd_finish_request(temp_dir): + # fastcgi_finish_request() ends the request while the script keeps + # running. The router then puts the next request into the released + # memory. The module read the target index after the script ended, so + # it kept the index of that next request as the last target. Its next + # request for that target then skipped chdir(). The next request is + # still running when the index is read, so libunit has not released or + # filled that memory, and the test fails on the old code in release + # builds too. Signal files order the requests, so the test does not + # depend on timing. + sig = Path(f'{temp_dir}/sig') + sig.mkdir() + sig.chmod(0o777) + + script = """ Date: Sat, 3 Oct 2026 15:01:28 +0200 Subject: [PATCH 334/391] http: answer "Expect: 100-continue" The router never sent "100 Continue". A client that sent "Expect: 100-continue" waited for its own timeout before it sent the body. For curl, that timeout is 1 second. curl and Guzzle send the field for large uploads, so each such upload took about 1 second longer. The router reads the body before it routes the request. So nxt_h1p_request_body_read() now sends "HTTP/1.1 100 Continue" when all of these are true: - The request is HTTP/1.1, and its Expect value is "100-continue" in any case. Guzzle sends "100-Continue". - The request has a body: a Content-Length above 0, or chunked. - The router must wait for more of the body. A part of the body that came with the header does not stop the 100, as in nginx. That part can be only chunk framing, such as "5\r\n", and the client can still wait for the 100. - No 100 was sent for this request yet. The flag is cleared when the 100 is queued, so a request gets one 100 at most. The 100 goes out on the normal write path, with send_timeout. The body read starts only after the 100 is sent, so the 100 cannot come after any part of the response. A write error or a send timeout closes the request, as for a response. $body_bytes_sent does not count the 100. A request that the router refuses from its header gets its final status with no 100. This is a Content-Length over max_body_size (413), and the Transfer-Encoding errors (400, 411, 501). The router ignores other expectation values, as nginx does. RFC 9110, section 10.1.1, allows this or a 417. The proxy no longer forwards Expect to the upstream. The router has the whole body before it connects, and it sends the header and the body together. nginx removes the field in the same way. An application still gets the field. Closes https://github.com/freeunitorg/freeunit/issues/533 Co-Authored-By: Claude Opus 5.5 --- CHANGES | 9 + docs/changes.xml | 12 ++ src/nxt_h1proto.c | 136 ++++++++++++- src/nxt_h1proto.h | 7 +- test/test_http_expect.py | 400 +++++++++++++++++++++++++++++++++++++ test/test_proxy_interim.py | 13 +- 6 files changed, 571 insertions(+), 6 deletions(-) create mode 100644 test/test_http_expect.py diff --git a/CHANGES b/CHANGES index fe1ecf2b1..6568641f4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: the router did not answer "Expect: 100-continue". A client + that sent it, as curl and Guzzle do for large uploads, waited for its + own timeout before it sent the body. For curl, that timeout is 1 + second. The router now sends "100 Continue" before it waits for the + body, also when a part of the body came with the header. A request + that the router refuses from its header gets its final status with no + 100. A proxied request no longer carries the Expect field to the + upstream. + *) Bugfix: the PHP module read the target index of a request after the request was released. libunit fills released request memory with 0xA5, so the module called chdir(2) on every request to a "script" diff --git a/docs/changes.xml b/docs/changes.xml index 240c7922a..ebd67e4d6 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,18 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +the router did not answer "Expect: 100-continue". A client that sent it, as +curl and Guzzle do for large uploads, waited for its own timeout before it +sent the body. For curl, that timeout is 1 second. The router now sends +"100 Continue" before it waits for the body, also when a part of the body +came with the header. A request that the router refuses from its header gets +its final status with no 100. A proxied request no longer carries the Expect +field to the upstream. + + + the PHP module read the target index of a request after the request was diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index cda6f7e0a..527afccec 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -43,7 +43,13 @@ static nxt_int_t nxt_h1p_websocket_version(void *ctx, nxt_http_field_t *field, uintptr_t data); static nxt_int_t nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data); +static nxt_int_t nxt_h1p_expect(void *ctx, nxt_http_field_t *field, + uintptr_t data); static void nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r); +static nxt_int_t nxt_h1p_request_continue(nxt_task_t *task, + nxt_h1proto_t *h1p); +static void nxt_h1p_conn_continue_sent(nxt_task_t *task, void *obj, + void *data); static void nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_request_local_addr(nxt_task_t *task, nxt_http_request_t *r); @@ -120,6 +126,7 @@ static const nxt_conn_state_t nxt_h1p_shutdown_state; #endif static const nxt_conn_state_t nxt_h1p_idle_state; static const nxt_conn_state_t nxt_h1p_header_parse_state; +static const nxt_conn_state_t nxt_h1p_continue_state; static const nxt_conn_state_t nxt_h1p_read_body_state; static const nxt_conn_state_t nxt_h1p_request_send_state; static const nxt_conn_state_t nxt_h1p_timeout_response_state; @@ -167,6 +174,7 @@ static nxt_http_field_proc_t nxt_h1p_fields[] = { { nxt_string("Sec-WebSocket-Version"), &nxt_h1p_websocket_version, 0 }, { nxt_string("Transfer-Encoding"), &nxt_h1p_transfer_encoding, 0 }, + { nxt_string("Expect"), &nxt_h1p_expect, 0 }, { nxt_string("Host"), &nxt_http_request_host, 0 }, { nxt_string("Cookie"), &nxt_http_request_field, @@ -927,6 +935,33 @@ nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data) } +/* + * RFC 9110, 10.1.1. The router meets "100-continue" itself and ignores any + * other expectation, as nginx does. An HTTP/1.0 client gets no 100. The + * proxy has the whole body before it connects, so it does not forward the + * field. The application still gets it. + */ + +static nxt_int_t +nxt_h1p_expect(void *ctx, nxt_http_field_t *field, uintptr_t data) +{ + nxt_http_request_t *r; + + r = ctx; + field->hopbyhop = 1; + + if (field->value_length == nxt_length("100-continue") + && nxt_memcasecmp(field->value, "100-continue", + nxt_length("100-continue")) == 0 + && nxt_h1p_is_http11(r->proto.h1)) + { + r->proto.h1->continue_pending = 1; + } + + return NXT_OK; +} + + static void nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) { @@ -1070,6 +1105,24 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) c = h1p->conn; c->read = b; + + /* + * The 100 goes out also if a part of the body came with the header, + * as in nginx. That part can be only chunk framing, and the client + * can still wait for the 100. Nothing is written before the body + * is read, so the 100 cannot follow a response. The flag is cleared + * when the 100 is queued, so a request gets one 100 at most. + */ + if (h1p->continue_pending) { + ret = nxt_h1p_request_continue(task, h1p); + if (nxt_slow_path(ret != NXT_OK)) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; + } + + return; + } + c->read_state = &nxt_h1p_read_body_state; nxt_conn_read(task->thread->engine, c); @@ -1097,6 +1150,84 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) } +/* + * Sends "100 Continue" on the normal write path. The body is read only + * after the 100 is sent. A write error or a send timeout closes the + * request, as for a response. + */ + +static nxt_int_t +nxt_h1p_request_continue(nxt_task_t *task, nxt_h1proto_t *h1p) +{ + nxt_buf_t *b; + nxt_conn_t *c; + + static const char continue_response[] = "HTTP/1.1 100 Continue\r\n\r\n"; + + nxt_debug(task, "h1p request continue"); + + c = h1p->conn; + + /* The connection pool frees the buffer also if it is never sent. */ + b = nxt_buf_mem_alloc(c->mem_pool, nxt_length(continue_response), 0); + if (nxt_slow_path(b == NULL)) { + return NXT_ERROR; + } + + b->mem.free = nxt_cpymem(b->mem.free, continue_response, + nxt_length(continue_response)); + + /* $body_bytes_sent does not count the 100. */ + h1p->sent_before_body += nxt_length(continue_response); + h1p->continue_pending = 0; + + c->write = b; + c->write_state = &nxt_h1p_continue_state; + + nxt_conn_write(task->thread->engine, c); + + return NXT_OK; +} + + +static const nxt_conn_state_t nxt_h1p_continue_state + nxt_aligned(64) = +{ + .ready_handler = nxt_h1p_conn_continue_sent, + .error_handler = nxt_h1p_conn_request_error, + + .timer_handler = nxt_h1p_conn_request_send_timeout, + .timer_value = nxt_h1p_conn_request_timer_value, + .timer_data = offsetof(nxt_socket_conf_t, send_timeout), + .timer_autoreset = 1, +}; + + +static void +nxt_h1p_conn_continue_sent(nxt_task_t *task, void *obj, void *data) +{ + nxt_conn_t *c; + nxt_event_engine_t *engine; + + c = obj; + + nxt_debug(task, "h1p conn continue sent"); + + engine = task->thread->engine; + + c->write = nxt_sendbuf_completion(task, &engine->fast_work_queue, c->write); + + if (c->write != NULL) { + nxt_conn_write(engine, c); + return; + } + + c->read_state = &nxt_h1p_read_body_state; + + nxt_conn_read(engine, c); +} + + static const nxt_conn_state_t nxt_h1p_read_body_state nxt_aligned(64) = { @@ -1540,7 +1671,8 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, header->mem.free = p; - h1p->header_size = nxt_buf_mem_used_size(&header->mem); + /* $body_bytes_sent does not count the header, or a 100 sent before. */ + h1p->sent_before_body += nxt_buf_mem_used_size(&header->mem); c = h1p->conn; @@ -1745,7 +1877,7 @@ nxt_h1p_request_body_bytes_sent(nxt_task_t *task, nxt_http_proto_t proto) h1p = proto.h1; - sent = h1p->conn->sent - h1p->header_size; + sent = h1p->conn->sent - h1p->sent_before_body; return (sent > 0) ? sent : 0; } diff --git a/src/nxt_h1proto.h b/src/nxt_h1proto.h index bff3dd2c8..3b6ba1870 100644 --- a/src/nxt_h1proto.h +++ b/src/nxt_h1proto.h @@ -46,10 +46,15 @@ struct nxt_h1proto_s { uint8_t websocket_cont_expected; /* 1 bit */ uint8_t websocket_closed; /* 1 bit */ uint8_t websocket_text; /* 1 bit */ + uint8_t continue_pending; /* 1 bit */ nxt_h1p_ws_utf8_t websocket_utf8; - uint32_t header_size; + /* + * Bytes written before the body: a 100 (Continue) and the response + * header. $body_bytes_sent does not count them. + */ + uint32_t sent_before_body; nxt_http_field_t *websocket_key; nxt_h1p_websocket_timer_t *websocket_timer; diff --git a/test/test_http_expect.py b/test/test_http_expect.py new file mode 100644 index 000000000..ab03965c3 --- /dev/null +++ b/test/test_http_expect.py @@ -0,0 +1,400 @@ +"""Expect: 100-continue (RFC 9110, 10.1.1). + +The router sends "100 Continue" when it is about to wait for more of the body. +A part of the body that came with the header does not stop the 100, as in +nginx. A request that the router answers from its header alone gets its final +status with no 100. The tests use raw sockets, because the client must wait +for the 100 before it sends the body. +""" + +import socket +import ssl +import time + +import pytest + +from unit.applications.lang.python import ApplicationPython +from unit.applications.tls import ApplicationTLS +from unit.option import option + +prerequisites = {'modules': {'python': 'any'}} + +client = ApplicationPython() + +CONTINUE = b'HTTP/1.1 100 Continue\r\n\r\n' + +# The longest wait for a 100. The wait ends when the 100 arrives, so a long +# value costs nothing. It covers a slow sanitizer build. +CONTINUE_WAIT = 2 + +# A test that expects nothing to arrive waits this long. +SILENCE_WAIT = 0.5 + + +def app(name): + python_dir = f'{option.test_dir}/python' + + return { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "path": f'{python_dir}/{name}', + "working_directory": f'{python_dir}/{name}', + "module": "wsgi", + } + + +@pytest.fixture(autouse=True) +def setup_method_fixture(): + assert 'success' in client.conf( + { + "listeners": { + "*:8080": {"pass": "applications/mirror"}, + "*:8081": {"pass": "routes"}, + "*:8082": {"pass": "applications/header_fields"}, + }, + "routes": [{"action": {"proxy": "http://127.0.0.1:8082"}}], + "applications": { + "mirror": app('mirror'), + "header_fields": app('header_fields'), + }, + "settings": {"http": {"chunked_transform": True}}, + } + ) + + +def connect(port=8080, wrap=None): + sock = socket.create_connection(('127.0.0.1', port)) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if wrap is not None: + sock = wrap(sock) + + return sock + + +def head(expect='100-continue', length=5, version='1.1', extra=''): + """A POST header section with no body.""" + + request = f'POST / HTTP/{version}\r\nHost: localhost\r\n' + + if expect is not None: + request += f'Expect: {expect}\r\n' + + if length is not None: + request += f'Content-Length: {length}\r\n' + + return (request + extra + '\r\n').encode() + + +def recv_for(sock, timeout, size=None): + """What arrives within timeout seconds, up to size bytes or EOF.""" + + data = b'' + end = time.monotonic() + timeout + + while size is None or len(data) < size: + left = end - time.monotonic() + if left <= 0: + break + + sock.settimeout(left) + + try: + part = sock.recv(4096) + except socket.timeout: + break + + if not part: + break + + data += part + + return data + + +def recv_continue(sock): + return recv_for(sock, CONTINUE_WAIT, len(CONTINUE)) + + +def recv_response(sock): + """One response framed by Content-Length: status line, fields, body.""" + + data = b'' + sock.settimeout(10) + + while b'\r\n\r\n' not in data: + part = sock.recv(4096) + assert part, f'closed before the header: {data!r}' + data += part + + header, _, body = data.partition(b'\r\n\r\n') + lines = header.decode().split('\r\n') + + fields = {} + for line in lines[1:]: + name, _, value = line.partition(':') + fields[name.strip().lower()] = value.strip() + + length = int(fields.get('content-length', 0)) + + while len(body) < length: + part = sock.recv(4096) + assert part, f'closed in the body: {body!r}' + body += part + + return lines[0], fields, body + + +def recv_all(sock): + return recv_for(sock, 10) + + +@pytest.mark.parametrize('expect', ['100-continue', '100-Continue']) +def test_expect_continue(expect): + with connect() as sock: + sock.sendall(head(expect, extra='Connection: close\r\n')) + + assert recv_continue(sock) == CONTINUE, 'no 100 within CONTINUE_WAIT' + + sock.sendall(b'hello') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'hello' + + +def test_expect_continue_chunked(): + with connect() as sock: + sock.sendall( + head( + length=None, + extra='Transfer-Encoding: chunked\r\nConnection: close\r\n', + ) + ) + + assert recv_continue(sock) == CONTINUE, 'no 100 within CONTINUE_WAIT' + + sock.sendall(b'5\r\nhello\r\n0\r\n\r\n') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'hello' + + +def test_expect_continue_keepalive(): + """Each request on a connection gets its own 100. The expectation of one + request does not carry over to the next one.""" + + with connect() as sock: + + for body in (b'hello', b'second'): + sock.sendall(head(length=len(body))) + + assert recv_continue(sock) == CONTINUE, f'no 100 before {body!r}' + + sock.sendall(body) + + status, _, got = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert got == body + + sock.sendall(head(expect=None, extra='Connection: close\r\n')) + + assert recv_for(sock, SILENCE_WAIT) == b'', '100 without Expect' + + sock.sendall(b'third') + + data = recv_all(sock) + assert data.startswith(b'HTTP/1.1 200'), data + assert data.endswith(b'\r\n\r\nthird'), data + + +def test_expect_continue_body_bytes_sent(wait_for_record): + """$body_bytes_sent does not count the 100.""" + + assert 'success' in client.conf( + { + 'path': f'{option.temp_dir}/access.log', + 'format': '$uri $status $body_bytes_sent', + }, + 'access_log', + ) + + with connect() as sock: + sock.sendall( + b'POST /bbs HTTP/1.1\r\nHost: localhost\r\n' + b'Expect: 100-continue\r\nContent-Length: 5\r\n' + b'Connection: close\r\n\r\n' + ) + + assert recv_continue(sock) == CONTINUE, 'no 100 within CONTINUE_WAIT' + + sock.sendall(b'hello') + recv_all(sock) + + found = wait_for_record(r'^/bbs 200 (\d+)$', 'access.log') + assert found is not None, 'no access log record' + assert found.group(1) == '5', '$body_bytes_sent' + + +def test_expect_continue_proxy(): + """The router meets the expectation, so a proxied upstream does not get + the field. An application gets it.""" + + for port, seen in ((8082, True), (8081, False)): + with connect(port) as sock: + sock.sendall(head(extra='Connection: close\r\n')) + + assert recv_continue(sock) == CONTINUE, f'no 100 on {port}' + + sock.sendall(b'hello') + + status, fields, _ = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + + names = fields['all-headers'].split(',') + assert ('HTTP_EXPECT' in names) == seen, f'{port}: {names}' + + +def test_expect_continue_tls(): + if not option.available['modules'].get('openssl'): + pytest.skip('requires openssl') + + tls = ApplicationTLS() + tls.certificate() + + assert 'success' in client.conf( + { + "pass": "applications/mirror", + "tls": {"certificate": "default"}, + }, + 'listeners/*:8080', + ) + + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + with connect(wrap=context.wrap_socket) as sock: + sock.sendall(head(extra='Connection: close\r\n')) + + assert recv_continue(sock) == CONTINUE, 'no 100 within CONTINUE_WAIT' + + sock.sendall(b'hello') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'hello' + + +@pytest.mark.parametrize( + 'request_head', + [ + # RFC 9110: an HTTP/1.0 expectation is ignored. + head(version='1.0', extra='Connection: close\r\n'), + # Only "100-continue" is known; the others are ignored, as nginx + # does. + head(expect='100-continue-please', extra='Connection: close\r\n'), + head(expect='foo', extra='Connection: close\r\n'), + ], + ids=['http10', 'other-value', 'unknown'], +) +def test_expect_continue_ignored(request_head): + with connect() as sock: + sock.sendall(request_head) + + assert recv_for(sock, SILENCE_WAIT) == b'', 'answered before the body' + + sock.sendall(b'hello') + + data = recv_all(sock) + assert data.startswith(b'HTTP/1.1 200'), data + assert CONTINUE not in data, data + + +@pytest.mark.parametrize( + 'request_head, status', + [ + # No body. + (head(length=None, extra='Connection: close\r\n'), 200), + (head(length=0, extra='Connection: close\r\n'), 200), + # Rejected from the header alone. + (head(length=None, extra='Transfer-Encoding: gzip\r\n'), 501), + (head(extra='Transfer-Encoding: chunked\r\n'), 400), + ], + ids=['no-length', 'zero-length', 'te-unsupported', 'te-and-length'], +) +def test_expect_continue_final_status(request_head, status): + """The router answers from the header alone, with no 100.""" + + with connect() as sock: + sock.sendall(request_head) + + data = recv_all(sock) + assert data.startswith(f'HTTP/1.1 {status}'.encode()), data + assert CONTINUE not in data, data + + +def test_expect_continue_max_body_size(): + """A body over max_body_size gets 413 at once, with no 100.""" + + assert 'success' in client.conf('10', 'settings/http/max_body_size') + + with connect() as sock: + sock.sendall(head(length=11)) + + data = recv_all(sock) + assert data.startswith(b'HTTP/1.1 413'), data + assert CONTINUE not in data, data + + +def test_expect_continue_length_required(): + """With chunked_transform off, a chunked body gets 411 at once, with no + 100.""" + + assert 'success' in client.conf('false', 'settings/http/chunked_transform') + + with connect() as sock: + sock.sendall(head(length=None, extra='Transfer-Encoding: chunked\r\n')) + + data = recv_all(sock) + assert data.startswith(b'HTTP/1.1 411'), data + assert CONTINUE not in data, data + + +def test_expect_continue_part_of_body(): + """A part of the body comes with the header. The router sends the 100 + while it waits for the rest, as nginx does.""" + + with connect() as sock: + sock.sendall(head(length=10, extra='Connection: close\r\n') + b'hello') + + assert recv_continue(sock) == CONTINUE, 'no 100 after part of the body' + + sock.sendall(b'world') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'helloworld' + + +def test_expect_continue_chunk_framing(): + """Only the framing of the first chunk comes with the header. The client + waits for the 100 before it sends the chunk data.""" + + with connect() as sock: + sock.sendall( + head( + length=None, + extra='Transfer-Encoding: chunked\r\nConnection: close\r\n', + ) + + b'5\r\n' + ) + + assert recv_continue(sock) == CONTINUE, 'no 100 after chunk framing' + + sock.sendall(b'hello\r\n0\r\n\r\n') + + status, _, body = recv_response(sock) + assert status.startswith('HTTP/1.1 200'), status + assert body == b'hello' diff --git a/test/test_proxy_interim.py b/test/test_proxy_interim.py index 20ce7dba3..6ac82fb6a 100644 --- a/test/test_proxy_interim.py +++ b/test/test_proxy_interim.py @@ -293,8 +293,9 @@ def test_proxy_interim_consumed(target): def test_proxy_interim_expect_continue(): - """The upstream answers Expect with 100; the client gets only the final - response.""" + """The upstream sends a 100 for /expect, and the proxy drops it. The + client gets one 100 only: the router sends it before it reads the rest + of the body.""" body = 'x' * 4096 @@ -307,7 +308,13 @@ def test_proxy_interim_expect_continue(): '\r\n' + body ) - check(raw, body=f'got {len(body)}') + # The first read holds at most header_buffer_size (2048) bytes, so the + # router always waits for the rest of the body. + router_continue = 'HTTP/1.1 100 Continue\r\n\r\n' + + assert raw.startswith(router_continue), f'no 100 from the router: {raw!r}' + + check(raw[len(router_continue) :], body=f'got {len(body)}') def test_proxy_interim_fields_not_leaked(): From d6ea31853eb5a3301976b3e20d09905f0db6bbbb Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 3 Oct 2026 14:20:45 +0200 Subject: [PATCH 335/391] http: end a chunked response header at once The router sent the header of a chunked HTTP/1.1 response without the empty line that ends it. The first chunk header added that line: it was "\r\n%xO\r\n" in nxt_h1p_chunk_create(). The last chunk of an empty body, "\r\n0\r\n\r\n", added it too. If an application sent the header and then waited before its body, the client had all header lines but not the end of the header. It could not use the status or the fields until the first body bytes came. An event stream is an example. Now nxt_h1p_request_header_send() ends every header with its CRLF. The first chunk header and the last chunk of an empty body no longer start with CRLF. Later chunk headers, and the last chunk after data, keep it, because it ends the data of the previous chunk. The new bit chunk_sent in nxt_h1proto_t records that a chunk was sent. It is reset with the other request fields on a keep-alive connection. The bytes of a complete response do not change. Only the time when the last two bytes of the header are sent changes. h1p->header_size now includes these two bytes, so $body_bytes_sent is 2 less for a chunked response. One cut response changes: if a proxied server closes before the first chunk, the client now gets the whole header and then the close. Before, it got the header without its end. The last chunk is missing in both cases. test_asgi_application_header_before_body checks that the end of the header comes before the body, the last chunk of an empty body, and a second response on a keep-alive connection. test_access_log_body_bytes_sent_chunked checks $body_bytes_sent for a chunked response. It is 20 for a 10-byte body and 5 for an empty body. The old router logged 22 and 7. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 8 +++++ docs/changes.xml | 11 ++++++ src/nxt_h1proto.c | 38 ++++++++++++++++----- src/nxt_h1proto.h | 1 + test/python/header_then_body/asgi.py | 27 +++++++++++++++ test/test_access_log.py | 30 ++++++++++++++++ test/test_asgi_application.py | 51 ++++++++++++++++++++++++++++ 7 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 test/python/header_then_body/asgi.py diff --git a/CHANGES b/CHANGES index 6568641f4..aa3f34666 100644 --- a/CHANGES +++ b/CHANGES @@ -18,6 +18,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 fastcgi_finish_request() when the released memory already held a request for another target. The bug had appeared in 1.18.0. + *) Bugfix: the router sent the header of a chunked HTTP/1.1 response + without the empty line that ends it, and sent that line with the + first body bytes. If an application sent the header and then + waited before its body, the client could not use the header until + the body began. Now the header ends at once. A complete response + has the same bytes as before. $body_bytes_sent no longer counts the + empty line. + *) Change: the main process now writes the state files in a short-lived child process, so a slow fsync(2) no longer delays the start of application processes after a reconfiguration, or diff --git a/docs/changes.xml b/docs/changes.xml index ebd67e4d6..e9235af08 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -33,6 +33,17 @@ held a request for another target. The bug had appeared in 1.18.0. + + +the router sent the header of a chunked HTTP/1.1 response without the empty +line that ends it, and sent that line with the first body bytes. If an +application sent the header and then waited before its body, the client could +not use the header until the body began. Now the header ends at once. A +complete response has the same bytes as before. $body_bytes_sent no longer +counts the empty line. + + + the main process now writes the state files in a short-lived child diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 527afccec..66179255e 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -1587,8 +1587,6 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, { h1p->chunked = 1; size += nxt_length(chunked); - /* Trailing CRLF will be added by the first chunk header. */ - size -= nxt_length("\r\n"); } } else if (!r->no_body) { @@ -1663,12 +1661,14 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, if (h1p->chunked) { p = nxt_cpymem(p, chunked, nxt_length(chunked)); - /* Trailing CRLF will be added by the first chunk header. */ - - } else { - *p++ = '\r'; *p++ = '\n'; } + /* + * The header ends here, also for a chunked response. Thus the client + * can use the header before the first body bytes come. + */ + *p++ = '\r'; *p++ = '\n'; + header->mem.free = p; /* $body_bytes_sent does not count the header, or a 100 sent before. */ @@ -1799,15 +1799,24 @@ nxt_h1p_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) } +/* + * The data of a chunk ends with CRLF. This CRLF starts the next chunk + * header or the last chunk. Before the first chunk there is no data to end: + * the response header ends with its own CRLF. + */ + static nxt_buf_t * nxt_h1p_chunk_create(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) { + u_char *p; nxt_off_t size; nxt_buf_t *b, **prev, *header, *tail; + nxt_h1proto_t *h1p; const size_t chunk_size = 2 * nxt_length("\r\n") + NXT_OFF_T_HEXLEN; static const char tail_chunk[] = "\r\n0\r\n\r\n"; + h1p = r->proto.h1; size = 0; prev = &out; @@ -1846,6 +1855,11 @@ nxt_h1p_chunk_create(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) nxt_memcpy(tail->mem.free, tail_chunk, sizeof(tail_chunk)); tail->mem.free += nxt_length(tail_chunk); + if (!h1p->chunk_sent && size == 0) { + /* No chunk data comes before the last chunk. */ + tail->mem.pos += nxt_length("\r\n"); + } + break; } @@ -1863,8 +1877,16 @@ nxt_h1p_chunk_create(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) } header->next = out; - header->mem.free = nxt_sprintf(header->mem.free, header->mem.end, - "\r\n%xO\r\n", size); + p = header->mem.free; + + if (h1p->chunk_sent) { + *p++ = '\r'; *p++ = '\n'; + } + + header->mem.free = nxt_sprintf(p, header->mem.end, "%xO\r\n", size); + + h1p->chunk_sent = 1; + return header; } diff --git a/src/nxt_h1proto.h b/src/nxt_h1proto.h index 3b6ba1870..1137db99e 100644 --- a/src/nxt_h1proto.h +++ b/src/nxt_h1proto.h @@ -37,6 +37,7 @@ struct nxt_h1proto_s { uint8_t large_buffer_slot; uint8_t keepalive; /* 1 bit */ uint8_t chunked; /* 1 bit */ + uint8_t chunk_sent; /* 1 bit */ uint8_t websocket; /* 1 bit */ uint8_t connection_upgrade; /* 1 bit */ uint8_t upgrade_websocket; /* 1 bit */ diff --git a/test/python/header_then_body/asgi.py b/test/python/header_then_body/asgi.py new file mode 100644 index 000000000..3cf29b4e9 --- /dev/null +++ b/test/python/header_then_body/asgi.py @@ -0,0 +1,27 @@ +import asyncio +from urllib.parse import parse_qs + + +async def application(scope, receive, send): + assert scope['type'] == 'http' + + query = parse_qs(scope['query_string'].decode()) + + delay = int(query.get('delay', ['0'])[0]) + body = query.get('body', [''])[0].encode() + + await send( + { + 'type': 'http.response.start', + 'status': 200, + 'headers': [(b'x-header', b'1')], + } + ) + + # Unit sends the response header for a body message without "body". + # An empty "body" sends nothing. + await send({'type': 'http.response.body', 'more_body': True}) + + await asyncio.sleep(delay) + + await send({'type': 'http.response.body', 'body': body}) diff --git a/test/test_access_log.py b/test/test_access_log.py index 6932bd57f..ce36c42a2 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -466,6 +466,36 @@ def test_access_log_variables(wait_for_record): ), '$body_bytes_sent' +def test_access_log_body_bytes_sent_chunked(wait_for_record): + # The application sets no Content-Length, so the response is chunked. + # $body_bytes_sent counts the chunk framing. It does not count the + # empty line that ends the header. + load('chunked') + set_format('$uri $body_bytes_sent') + + body = '0123456789' + + resp = client.post(url='/data', body=body) + assert resp['status'] == 200 + assert resp['headers']['Transfer-Encoding'] == 'chunked' + assert resp['body'] == body + + # "a\r\n0123456789\r\n0\r\n\r\n" is 20 bytes. + record = wait_for_record(r'^/data \d+$', 'access.log') + assert record is not None, 'data record' + assert record.group(0) == '/data 20', 'chunked body' + + # An empty body has only the last chunk: "0\r\n\r\n" is 5 bytes. + resp = client.get(url='/empty') + assert resp['status'] == 200 + assert resp['headers']['Transfer-Encoding'] == 'chunked' + assert resp['body'] == '' + + record = wait_for_record(r'^/empty \d+$', 'access.log') + assert record is not None, 'empty record' + assert record.group(0) == '/empty 5', 'empty chunked body' + + def test_access_log_if(search_in_file, wait_for_record): load('empty') set_format('$uri') diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index c0888f265..c7d4a52ac 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -196,6 +196,57 @@ def test_asgi_application_204_transfer_encoding(): ), '204 header transfer encoding' +def get_header_then_body(url): + # The application sleeps after it sends the header. Read for less + # time than the sleep: what arrives in that time came before the body. + # Then read the rest of the response. + sock = client.get(url=url, no_recv=True) + + early = client.recvall(sock, read_timeout=0.5).decode() + rest = client.recvall(sock).decode() + sock.close() + + return early, rest + + +def test_asgi_application_header_before_body(): + client.load('header_then_body') + + resp = client.get(url='/?body=data') + assert resp['status'] == 200, 'start the application' + + # The response has no Content-Length, so it is chunked. The empty + # line that ends the header must come with the header, not with the + # first chunk. + early, rest = get_header_then_body('/?delay=2&body=data') + + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'status line' + assert 'x-header: 1\r\n' in early, 'header field' + assert 'Transfer-Encoding: chunked\r\n' in early, 'chunked' + assert early.endswith('\r\n\r\n'), 'header end before the body' + assert rest == '4\r\ndata\r\n0\r\n\r\n', 'chunks after the sleep' + + # An empty body: the last chunk is the first chunk. + early, rest = get_header_then_body('/?delay=2') + + assert early.endswith('\r\n\r\n'), 'empty body: header end' + assert rest == '0\r\n\r\n', 'empty body: last chunk only' + + # Two responses on one connection. The first chunk of the second + # response has no CRLF before it either. + one = 'Transfer-Encoding: chunked\r\n\r\n4\r\ndata\r\n0\r\n\r\n' + + raw = client.http( + b'GET /?body=data HTTP/1.1\r\nHost: localhost\r\n\r\n' + b'GET /?body=data HTTP/1.1\r\nHost: localhost\r\n' + b'Connection: close\r\n\r\n', + raw=True, + raw_resp=True, + ) + + assert raw.count(one) == 2, 'keep-alive' + + def test_asgi_application_shm_ack_handle(): # Minimum possible limit shm_limit = 10 * 1024 * 1024 From ac2f2d294ecc970daa74b60b2540e9334c7f14e0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Fri, 2 Oct 2026 22:52:26 +0200 Subject: [PATCH 336/391] php: send the response header on flush() The PHP module registered no flush handler, so flush() did nothing. If a script had no output yet, or its output was in an output buffer, the client got the response header only at the end of the request. headers_sent() returned false after flush(). Now nxt_php_flush() calls sapi_send_headers() if the header was not sent yet. Output needs no flush here: nxt_php_unbuffered_write() sends each write to the router at once. As in mod_php, flush() does not empty the output buffers of PHP; ob_flush() does. The commit "http: end a chunked response header at once" makes the router send the empty line that ends a chunked response header with the header. Thus after flush(), a client of a chunked HTTP/1.1 response has the complete header before the first body bytes. The tests check that this empty line comes before the sleep in the script ends. The module has had no flush handler since 1.4, when it moved to libunit. test_php_application_flush_after_finish calls flush() twice after fastcgi_finish_request(). The client must get only the output from before that call, in a complete response. unit.log must have no alert, and the same process must serve the next request. The check of ctx->req in nxt_php_flush() is not the only guard today. fastcgi_finish_request() sends the header, so SG(headers_sent) is 1 and the handler does not call sapi_send_headers(). Also, nxt_php_send_headers() checks ctx->req itself. So the test passes without the check. It fails if nxt_php_flush() dereferences ctx->req without the check, for example in nxt_unit_response_is_sent(): the process then exits on signal 11. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 8 ++ docs/changes.xml | 11 +++ src/nxt_php_sapi.c | 35 ++++++++- test/php/flush/index.php | 20 +++++ test/php/flush_after_finish/index.php | 19 +++++ test/test_php_application.py | 103 ++++++++++++++++++++++++++ 6 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 test/php/flush/index.php create mode 100644 test/php/flush_after_finish/index.php diff --git a/CHANGES b/CHANGES index aa3f34666..d6e3749b8 100644 --- a/CHANGES +++ b/CHANGES @@ -18,6 +18,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 fastcgi_finish_request() when the released memory already held a request for another target. The bug had appeared in 1.18.0. + *) Bugfix: flush() in a PHP application did not send the response + header. If the application had no output yet, or its output was + in an output buffer, the client got the header only at the end of + the request, and headers_sent() returned false after flush(). Now + flush() sends the header. As in mod_php, flush() does not empty + the output buffers; ob_flush() does. The PHP module has had no + flush handler since 1.4. + *) Bugfix: the router sent the header of a chunked HTTP/1.1 response without the empty line that ends it, and sent that line with the first body bytes. If an application sent the header and then diff --git a/docs/changes.xml b/docs/changes.xml index e9235af08..d3ab19791 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -33,6 +33,17 @@ held a request for another target. The bug had appeared in 1.18.0. + + +flush() in a PHP application did not send the response header. If the +application had no output yet, or its output was in an output buffer, the +client got the header only at the end of the request, and headers_sent() +returned false after flush(). Now flush() sends the header. As in mod_php, +flush() does not empty the output buffers; ob_flush() does. The PHP module +has had no flush handler since 1.4. + + + the router sent the header of a chunked HTTP/1.1 response without the empty diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 19594bc8f..f9a823fc7 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -149,6 +149,7 @@ static size_t nxt_php_read_post(char *buffer, size_t count_bytes TSRMLS_DC); static int nxt_php_unbuffered_write(const char *str, uint str_length TSRMLS_DC); static int nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC); #endif +static void nxt_php_flush(void *server_context); #ifdef NXT_PHP7 @@ -306,7 +307,7 @@ static sapi_module_struct nxt_php_sapi_module = NULL, /* deactivate */ nxt_php_unbuffered_write, /* unbuffered write */ - NULL, /* flush */ + nxt_php_flush, /* flush */ NULL, /* get uid */ NULL, /* getenv */ @@ -1397,6 +1398,38 @@ nxt_php_unbuffered_write(const char *str, uint str_length TSRMLS_DC) } +/* + * PHP calls this for flush(), and after each write when implicit_flush is + * on. nxt_php_unbuffered_write() sends each write to the router at once. + * Thus only the response header can wait here. This function does not + * empty the output buffers of PHP; ob_flush() does that, as in mod_php. + */ + +static void +nxt_php_flush(void *server_context) +{ + nxt_php_run_ctx_t *ctx; + + ctx = server_context; + + /* + * Module startup has no context. fastcgi_finish_request() leaves the + * context without a request. + */ + if (ctx == NULL || ctx->req == NULL) { + return; + } + + /* + * If the send fails, SG(headers_sent) stays 0. The next output then + * tries again and handles the error, as it does without flush(). + */ + if (!SG(headers_sent)) { + (void) sapi_send_headers(TSRMLS_C); + } +} + + static int nxt_php_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC) { diff --git a/test/php/flush/index.php b/test/php/flush/index.php new file mode 100644 index 000000000..60dff035b --- /dev/null +++ b/test/php/flush/index.php @@ -0,0 +1,20 @@ + diff --git a/test/php/flush_after_finish/index.php b/test/php/flush_after_finish/index.php new file mode 100644 index 000000000..429949f4c --- /dev/null +++ b/test/php/flush_after_finish/index.php @@ -0,0 +1,19 @@ + diff --git a/test/test_php_application.py b/test/test_php_application.py index 6f2fc812d..935ba95ed 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -238,6 +238,109 @@ def test_php_application_fastcgi_finish_request_2(findall, unit_pid): assert len(errs) == 0, 'no error' +def get_flushed(method, url, **kwargs): + # The "flush" app sleeps after flush(). Read for less time than the + # sleep: what arrives in that time is what flush() sent. Then read + # the rest of the response. + sock = client.http(method, url=url, no_recv=True, **kwargs) + + early = client.recvall(sock, read_timeout=0.5).decode() + rest = client.recvall(sock).decode() + sock.close() + + resp = client._resp_to_dict(early + rest) + + if resp['headers'].get('Transfer-Encoding') == 'chunked': + resp['body'] = client._parse_chunked_body(resp['body']).decode() + + return early, resp + + +def test_php_application_flush_headers(): + client.load('flush', options={'admin': {'output_buffering': '0'}}) + + resp = client.get() + assert resp['status'] == 200, 'start the application' + assert resp['headers']['X-OB-Level'] == '0', 'no output buffer' + + early, resp = get_flushed('GET', '/?sleep=2') + + assert resp['status'] == 200 + assert resp['body'] == 'part2;headers_sent=true;part3;', 'headers_sent()' + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'status line before sleep' + assert 'X-Flush: 1\r\n' in early, 'headers before sleep' + assert 'Transfer-Encoding: chunked\r\n' in early, 'chunked' + assert early.endswith('\r\n\r\n'), 'header end before sleep' + assert 'part2' not in early, 'no body before sleep' + + # A response to HTTP/1.0 is not chunked. The connection close ends + # its body. + early, resp = get_flushed('GET', '/?sleep=2', http_10=True) + + assert resp['body'] == 'part2;headers_sent=true;part3;', 'HTTP/1.0 body' + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'HTTP/1.0 status line' + assert 'X-Flush: 1\r\n' in early, 'HTTP/1.0 headers' + assert early.endswith('\r\n\r\n'), 'HTTP/1.0 header end' + + early, resp = get_flushed('HEAD', '/?sleep=2') + + assert resp['status'] == 200 + assert resp['headers']['X-Flush'] == '1' + assert resp['body'] == '', 'HEAD body' + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'HEAD status line' + assert early.endswith('\r\n\r\n'), 'HEAD header end' + + +def test_php_application_flush_output_buffering(): + client.load('flush', options={'admin': {'output_buffering': '4096'}}) + + resp = client.get() + assert resp['status'] == 200, 'start the application' + assert resp['headers']['X-OB-Level'] == '1', 'output buffer' + + early, resp = get_flushed('GET', '/?sleep=2&part1') + + assert resp['status'] == 200 + assert ( + resp['body'] == 'part1;part2;headers_sent=true;part3;' + ), 'headers_sent()' + assert early.startswith('HTTP/1.1 200 OK\r\n'), 'status line before sleep' + assert 'X-Flush: 1\r\n' in early, 'headers before sleep' + + # flush() does not empty the output buffers of PHP, as in mod_php. + # ob_flush() does that. So only the header comes before the sleep. + assert early.endswith('\r\n\r\n'), 'header end before sleep' + assert 'part1' not in early, 'part1 stays in the output buffer' + + +def test_php_application_flush_after_finish(findall, wait_for_record): + # fastcgi_finish_request() ends the response and leaves the context + # without a request. The script then calls flush() twice. + client.load('flush_after_finish') + + resp = client.get() + assert resp['status'] == 200 + # The client parses the chunked body, so a cut body fails here. + assert resp['headers']['Transfer-Encoding'] == 'chunked' + assert resp['body'] == 'before;', 'only the output before the finish' + + pid = resp['headers']['X-Pid'] + + # The worker is busy until the script ends. Then it serves the next + # request. + resp = client.get() + assert resp['status'] == 200 + assert resp['body'] == 'before;', 'second request' + + assert findall(r'.+\[alert\].+') == [], 'no alert' + assert resp['headers']['X-Pid'] == pid, 'the same process' + + assert ( + wait_for_record(r'php message: flush after finish: done') is not None + ), 'the script ran past flush()' + assert findall(r'Error in fastcgi_finish_request') == [] + + def test_php_application_query_string_absent(): client.load('query_string') From 85c23dbbee77265828c72814a8f21ee5b01d94c2 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 06:31:02 +0200 Subject: [PATCH 337/391] regex: cap the match work of one regex test at 100,000 steps pcre2_match ran with no match context, so a pattern took the library default of 10,000,000 steps. That is about 180 ms on an 8 KiB subject. Each request now gets one match context with a match limit of 100,000 steps (NXT_REGEX_MATCH_LIMIT). A match that reaches the limit fails, and the request gets a 500 response. The library counts steps from zero again at each start position of a pattern that is not anchored. So the match limit alone does not bound one whole match. In PCRE2 10.42, "(?=a)(?:a|aa){0,14}[xy]" on a URI of 8,000 "a" bytes and an "x" took 30.4 s, and the response was 200. Such a pattern is now compiled a second time with PCRE2_AUTO_CALLOUT. The callout counts down a budget of NXT_REGEX_MATCH_LIMIT calls in the match object of the request, and returns PCRE2_ERROR_CALLOUT at zero. That stops the whole match. The same URI now gets a 500 after 4 ms. Callouts do not change the result of a match. An anchored pattern has one start position. It is compiled without callouts and costs the same as before. Measured on PCRE2 10.46, an unanchored "\.(css|js|png)$" took 212 ns per match before and 381 ns after. The limit applies to each regex test. N regex rules on one subject can use N times the work. The limit is the same for all routes. The match context uses the system allocator. PCRE2 10.30 to 10.40 allocate the backtracking frames with it and free them after each match, and a pool does not free memory. The request pool frees the context. The context also sets a heap limit of 16 MB on PCRE2 10.30 and later. A frame takes 128 bytes plus 16 for each capturing group, so the match limit already bounds frames for a pattern with few groups. The library default heap limit is about 20 GB. Route patterns in real use need 2 to about 8,200 steps. A pattern that reaches the limit still costs about 2 ms of CPU per request on an 8 KiB subject. A repeated group takes about 2 steps per byte. In PCRE2 10.42, "^/(a|b)+$" takes 16,386 steps on 8 KiB, and "^(a|b)+$" takes 131,076 steps on 64 KiB. So a subject much longer than 8 KiB can reach the limit. Such a subject needs a "large_header_buffer_size" above its default of 8192. An unanchored pattern made 2 to 3.5 callouts per step in the patterns measured, so its budget runs out sooner. PCRE 1 counts steps in the same way. The PCRE 1 path keeps a pcre_extra in the match object of each request, with the match limit and the callout data. It compiles a pattern that is not anchored with PCRE_AUTO_CALLOUT, and sets the global pcre_callout once in nxt_regex_compile(). PCRE 1 also nests calls on the stack, about 2 per byte for a repeated group, and the match limit does not bound them. A call uses 512 bytes in libpcre3 8.39 on x86-64. "^/(a|b)+$" on an 8 KiB subject stays far below the match limit, but nests 16,384 calls and needs 8 MB of stack. The unit test crashed there. A request matches on a router thread with the default glibc stack size: 8 MB, or 2 MB when the stack rlimit is unlimited. With the default header buffer of 8192 bytes, the router used just less than 8 MB. But one request crashed the router with SIGSEGV with an 8,000-byte header and a 2 MB stack, or with a 12,000-byte header and a "large_header_buffer_size" of 16384. These crashes were there before this change. The PCRE 1 path now also sets a recursion limit of 2,000 calls, about 1 MB of stack. That fits in both default stack sizes. A match that reaches the recursion limit fails in the same way as one that reaches the match limit, and the request gets a 500 response. So with PCRE 1, a repeated group on a subject of more than about 1,000 bytes can get a 500 response. A client can now cause a match error on each request. Before, the error log got the full subject, up to 8 KiB of client data, at the "error" level. A match that reaches a limit now logs a warning with the pattern and the subject length only. Other match errors log at most 64 bytes of the subject. The "types" rule of "compression" checked only for 0, so a match error was a match and the response was compressed. A match error is now not a match, so the response is not compressed. A client that refused identity gets a 406. The unit test and test_routing.py send a subject that needs about 2M steps. Without the limit the match does not fail. test_routing.py also checks the warning and that the subject is not in the log. Both also send 1000 "a" bytes and an "x" to "(?=a)(?:a|aa){0,12}[xy]". It takes 8,193 steps at each of 976 start positions, so only the callout budget stops it. Before, it matched after about 1 s. The unit test checks that the same pattern with "^" does not match. test_static_compression.py sends a file with a long media type that reaches the limit. It must not be compressed, and with "gzip, identity;q=0" it must get a 406. With PCRE 1, the unit test expects an error for "^/(a|b)+$" on 8 KiB, where it crashed before, and a match on 512 bytes. The new c-tests-pcre1 CI job builds with --no-pcre2 and runs the unit tests. No other job compiles src/nxt_pcre.c. Co-Authored-By: Claude Opus 5.5 Reviewed-by: GPT-6.1-Sol Reviewed-by: DeepSeek V4.1 Flash Harness: DeepSeek Harness Reviewed-by: Claude Fable 5.1 --- .github/workflows/build-test.yml | 27 +++++ CHANGES | 13 +++ auto/sources | 2 + docs/changes.xml | 16 +++ src/nxt_http_compression.c | 8 +- src/nxt_http_route.c | 11 +- src/nxt_pcre.c | 130 +++++++++++++++++---- src/nxt_pcre2.c | 195 ++++++++++++++++++++++++++++--- src/nxt_regex.h | 21 +++- src/test/nxt_regex_test.c | 160 +++++++++++++++++++++++++ src/test/nxt_tests.c | 3 + src/test/nxt_tests.h | 3 + test/test_routing.py | 54 +++++++++ test/test_static_compression.py | 60 ++++++++++ 14 files changed, 654 insertions(+), 49 deletions(-) create mode 100644 src/test/nxt_regex_test.c diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 5dc4a0cdc..ed028c334 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1028,6 +1028,33 @@ jobs: - name: Run libunit malformed port message test run: ./build/unit_msg_test + # The C unit-test suite with PCRE 1 instead of PCRE2. nxt_pcre.c has its + # own match limits, recursion limit and callout. No other job compiles + # it. ubuntu-24.04 is pinned because it still ships libpcre3-dev. + c-tests-pcre1: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v5 + + - name: Install packages + run: | + sudo apt-get -y update + sudo apt-get -y install build-essential libpcre3-dev + + - name: Configure unit (--no-pcre2 --tests) + run: ./configure --no-pcre2 --tests + + - name: Build unitd + run: make -j4 + + - name: Build C tests + run: make -j4 tests + + - name: Run C tests + run: sudo ./build/tests + # Same C unit-test suite as c-tests above, built with AddressSanitizer and # run with LeakSanitizer's default ASAN_OPTIONS (detect_leaks is on unless # something turns it off). Separate from sanitize.yml: that workflow runs diff --git a/CHANGES b/CHANGES index d6e3749b8..38dc36100 100644 --- a/CHANGES +++ b/CHANGES @@ -42,6 +42,19 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 stored. At exit, the main process waits for the store to finish. When the store fails, the main process logs an alert. + *) Bugfix: a regular expression in the configuration could run for up + to 10,000,000 match steps on one request. A match now stops after + 100,000 steps and the request gets a 500 response. The error log + then gets a warning with the pattern and the subject length, not + the subject. In "compression" "types", a match that stops is not a + match, and the response is not compressed. A repeated group takes + about 2 steps per byte. So a subject much longer than 8 KiB can + reach the limit and get a 500 response. Such a subject needs a + "large_header_buffer_size" above its default of 8192. With PCRE 1, + a match also stops at 2,000 nested calls and does not overflow the + thread stack. There, a repeated group on a subject of more than + about 1,000 bytes can get a 500 response. + *) Security: the router and libunit read records from shared memory that the other process can still write, without enough checks. A buffer that was not a whole number of records was read past its diff --git a/auto/sources b/auto/sources index bf7c7e011..e61217186 100644 --- a/auto/sources +++ b/auto/sources @@ -275,6 +275,8 @@ if [ "$NXT_REGEX" = "YES" ]; then else NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE_SRCS" fi + + NXT_TEST_SRCS="$NXT_TEST_SRCS src/test/nxt_regex_test.c" fi if [ "$NXT_HAVE_EPOLL" = "YES" -o "$NXT_TEST_BUILD_EPOLL" = "YES" ]; then diff --git a/docs/changes.xml b/docs/changes.xml index d3ab19791..2433b95cc 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -66,6 +66,22 @@ store to finish. When the store fails, the main process logs an alert. + + +a regular expression in the configuration could run for up to 10,000,000 +match steps on one request. A match now stops after 100,000 steps and the +request gets a 500 response. The error log then gets a warning with the +pattern and the subject length, not the subject. In "compression" "types", a +match that stops is not a match, and the response is not compressed. A +repeated group takes about 2 steps per byte. So a subject much longer than +8 KiB can reach the limit and get a 500 response. Such a subject needs a +"large_header_buffer_size" above its default of 8192. With PCRE 1, a match +also stops at 2,000 nested calls and does not overflow the thread stack. +There, a repeated group on a subject of more than about 1,000 bytes can get a +500 response. + + + when unitd ran as root and the control socket was given to another user or diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index eb1a2fe22..51fa9e0e9 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -1352,7 +1352,13 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) ret = nxt_http_route_test_rule(r, conf->mime_types_rule, mime_type.start, mime_type.length); - if (ret == 0) { + + /* + * NXT_ERROR is not a match. A regex can reach its match limit on a + * long media type. The response then takes the no-match path. + */ + + if (ret == 0 || nxt_slow_path(ret == NXT_ERROR)) { return identity_refused ? nxt_http_comp_not_acceptable(r) : NXT_OK; } } diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index ce3eb0a84..01db1ee94 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -2164,12 +2164,11 @@ nxt_http_route_pattern(nxt_http_request_t *r, nxt_http_route_pattern_t *pattern, if (pattern->regex) { if (r->regex_match == NULL) { /* - * Reuse one match-data struct across every pattern compiled - * against this request, so size it for the minimum ovector - * (one offset pair — the overall match). Captures are not - * consulted by the matcher. Passing 0 to PCRE2's - * pcre2_match_data_create() is undefined per the public - * docs; 1 is the documented minimum. + * One match object serves every pattern tested against this + * request. Its match data holds the minimum ovector: one + * offset pair, the overall match. The matcher does not read + * captures. pcre2_match_data_create() with 0 is undefined in + * the public docs; 1 is the documented minimum. */ r->regex_match = nxt_regex_match_create(r->mem_pool, 1); if (nxt_slow_path(r->regex_match == NULL)) { diff --git a/src/nxt_pcre.c b/src/nxt_pcre.c index 737fc7cf7..28e137af2 100644 --- a/src/nxt_pcre.c +++ b/src/nxt_pcre.c @@ -9,30 +9,46 @@ struct nxt_regex_s { - pcre *code; - pcre_extra *extra; - nxt_str_t pattern; + pcre *code; + nxt_str_t pattern; }; +/* + * The limits and the match data of one request. "extra" passes the + * structure to the callout, which counts down "budget". + */ + struct nxt_regex_match_s { - int ovecsize; - int ovec[]; + pcre_extra extra; + uint32_t budget; + int ovecsize; + int ovec[]; }; static void *nxt_pcre_malloc(size_t size); static void nxt_pcre_free(void *p); +static int nxt_pcre_callout(pcre_callout_block *block); static nxt_mp_t *nxt_pcre_mp; +/* + * The most nested calls that one match may make. PCRE 1 makes them on the + * stack, 512 bytes each in libpcre3 8.39 on x86-64. 2,000 calls use about + * 1 MB. A request matches on a router thread with the default glibc stack: + * 8 MB, or 2 MB when the stack rlimit is unlimited. + */ +#define NXT_PCRE_RECURSION_LIMIT 2000 + nxt_regex_t * nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) { - int erroffset; - char *pattern; - void *saved_malloc, *saved_free; - nxt_regex_t *re; + int erroffset; + char *pattern; + void *saved_malloc, *saved_free; + nxt_regex_t *re; + unsigned long options; err->offset = source->length; @@ -58,21 +74,41 @@ nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) nxt_pcre_mp = mp; re->code = pcre_compile(pattern, 0, &err->msg, &erroffset, NULL); - if (nxt_fast_path(re->code != NULL)) { -#if 0 - re->extra = pcre_study(re->code, PCRE_STUDY_JIT_COMPILE, &err->msg); - if (nxt_slow_path(re->extra == NULL && err->msg != NULL)) { - nxt_log_warn(thr->log, "pcre_study(%V) failed: %s", source, err->msg); - } -#else - re->extra = NULL; -#endif - } else { + /* + * The library counts match steps from zero again at each start position + * of a pattern that is not anchored. So the match limit does not bound + * one whole match of it. Such a pattern is compiled again with a + * callout before each item. One match may make at most + * NXT_REGEX_MATCH_LIMIT callouts in total. The callouts do not change + * the result. An anchored pattern has one start position and gets no + * callouts. + * + * PCRE_INFO_OPTIONS writes an unsigned long. + */ + + if (re->code != NULL + && pcre_fullinfo(re->code, NULL, PCRE_INFO_OPTIONS, &options) == 0 + && (options & PCRE_ANCHORED) == 0) + { + re->code = pcre_compile(pattern, PCRE_AUTO_CALLOUT, &err->msg, + &erroffset, NULL); + } + + if (nxt_slow_path(re->code == NULL)) { err->offset = erroffset; re = NULL; } + /* + * The callout function is global in PCRE 1. It is set here, before any + * match with the pattern. No other code in the process sets it. + */ + + if (pcre_callout != nxt_pcre_callout) { + pcre_callout = nxt_pcre_callout; + } + pcre_malloc = saved_malloc; pcre_free = saved_free; @@ -101,13 +137,38 @@ nxt_pcre_free(void *p) } +static int +nxt_pcre_callout(pcre_callout_block *block) +{ + nxt_regex_match_t *match; + + match = block->callout_data; + + /* A negative value stops the whole match with this value. */ + + if (match->budget == 0) { + return PCRE_ERROR_CALLOUT; + } + + match->budget--; + + return 0; +} + + nxt_regex_match_t * nxt_regex_match_create(nxt_mp_t *mp, size_t size) { nxt_regex_match_t *match; - match = nxt_mp_get(mp, sizeof(nxt_regex_match_t) + sizeof(int) * size); + match = nxt_mp_zget(mp, sizeof(nxt_regex_match_t) + sizeof(int) * size); if (nxt_fast_path(match != NULL)) { + match->extra.flags = PCRE_EXTRA_MATCH_LIMIT + | PCRE_EXTRA_MATCH_LIMIT_RECURSION + | PCRE_EXTRA_CALLOUT_DATA; + match->extra.match_limit = NXT_REGEX_MATCH_LIMIT; + match->extra.match_limit_recursion = NXT_PCRE_RECURSION_LIMIT; + match->extra.callout_data = match; match->ovecsize = size; } @@ -121,9 +182,34 @@ nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, { int ret; - ret = pcre_exec(re->code, re->extra, (const char *) subject, length, 0, 0, - match->ovec, match->ovecsize); + match->budget = NXT_REGEX_MATCH_LIMIT; + + ret = pcre_exec(re->code, &match->extra, (const char *) subject, length, + 0, 0, match->ovec, match->ovecsize); if (nxt_slow_path(ret < PCRE_ERROR_NOMATCH)) { + + /* + * A client can cause these errors on each request. So the log gets + * no subject bytes for a limit, and at most 64 for others. The + * callout returns PCRE_ERROR_CALLOUT when one match makes more + * than NXT_REGEX_MATCH_LIMIT callouts. + */ + + if (ret == PCRE_ERROR_MATCHLIMIT || ret == PCRE_ERROR_CALLOUT + || ret == PCRE_ERROR_RECURSIONLIMIT) + { + nxt_thread_log_error(NXT_LOG_WARN, + "pcre_exec() reached the %s limit on %uz " + "bytes using \"%V\"", + (ret == PCRE_ERROR_RECURSIONLIMIT) + ? "recursion" : "match", + length, &re->pattern); + + return NXT_ERROR; + } + + length = nxt_min(length, NXT_REGEX_LOG_SUBJECT); + nxt_thread_log_error(NXT_LOG_ERR, "pcre_exec() failed: %d on \"%*s\" using \"%V\"", ret, length, subject, &re->pattern); diff --git a/src/nxt_pcre2.c b/src/nxt_pcre2.c index cb51062c0..35e353dec 100644 --- a/src/nxt_pcre2.c +++ b/src/nxt_pcre2.c @@ -10,8 +10,22 @@ #include +/* + * The most heap memory in KiB that one match may use for backtracking + * frames. In PCRE2 10.42 on x86-64, a frame takes 128 bytes, plus 16 + * bytes for each capturing group. 100,000 frames of 160 bytes take about + * 16 MB. The match limit permits no more frames than that, so this limit + * can stop a pattern with several capturing groups first: three with + * PCRE2 10.42, two with 10.46, where a frame is 8 bytes larger. The + * library default is about 20 GB. + */ +#define NXT_PCRE2_HEAP_LIMIT 16384 + + static void *nxt_pcre2_malloc(PCRE2_SIZE size, void *memory_data); static void nxt_pcre2_free(void *p, void *memory_data); +static int nxt_pcre2_callout(pcre2_callout_block *block, void *data); +static void nxt_pcre2_match_cleanup(nxt_task_t *task, void *obj, void *data); struct nxt_regex_s { @@ -20,12 +34,26 @@ struct nxt_regex_s { }; +/* + * The match data and the match context of one request. The callout of the + * context gets the structure and counts down "budget". + */ + +struct nxt_regex_match_s { + pcre2_match_data *data; + pcre2_match_context *ctx; + uint32_t budget; +}; + + nxt_regex_t * nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) { int errcode; + uint32_t options; nxt_int_t ret; PCRE2_SIZE erroffset; + pcre2_code *code; nxt_regex_t *re; pcre2_general_context *general_ctx; pcre2_compile_context *compile_ctx; @@ -55,20 +83,40 @@ nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) re->code = pcre2_compile((PCRE2_SPTR) source->start, source->length, 0, &errcode, &erroffset, compile_ctx); if (nxt_slow_path(re->code == NULL)) { - err->offset = erroffset; + goto compile_fail; + } - ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, - ERR_BUF_SIZE); - if (ret < 0) { - (void) nxt_sprintf(err->msg, err->msg + ERR_BUF_SIZE, - "compilation failed with unknown " - "error code: %d%Z", errcode); + /* + * The library counts match steps from zero again at each start position + * of a pattern that is not anchored. So the match limit does not bound + * one whole match of it. Such a pattern is compiled again with a + * callout before each item. One match may make at most + * NXT_REGEX_MATCH_LIMIT callouts in total. The callouts do not change + * the result. An anchored pattern has one start position and gets no + * callouts. + */ + + if (pcre2_pattern_info(re->code, PCRE2_INFO_ALLOPTIONS, &options) == 0 + && (options & PCRE2_ANCHORED) == 0) + { + code = pcre2_compile((PCRE2_SPTR) source->start, source->length, + PCRE2_AUTO_CALLOUT, &errcode, &erroffset, + compile_ctx); + if (nxt_slow_path(code == NULL)) { + goto compile_fail; } - return NULL; + pcre2_code_free(re->code); + re->code = code; } #if 0 + /* + * The JIT honours the match limit and callouts but not the heap limit, + * and it has its own stack limit. This matters if the JIT is enabled + * again. + */ + errcode = pcre2_jit_compile(re, PCRE2_JIT_COMPLETE); if (nxt_slow_path(errcode != 0 && errcode != PCRE2_ERROR_JIT_BADOPTION)) { ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, @@ -85,6 +133,20 @@ nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) return re; +compile_fail: + + err->offset = erroffset; + + ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, + ERR_BUF_SIZE); + if (ret < 0) { + (void) nxt_sprintf(err->msg, err->msg + ERR_BUF_SIZE, + "compilation failed with unknown " + "error code: %d%Z", errcode); + } + + return NULL; + alloc_fail: err->offset = source->length; @@ -107,24 +169,88 @@ nxt_pcre2_free(void *p, void *mp) } +static int +nxt_pcre2_callout(pcre2_callout_block *block, void *data) +{ + nxt_regex_match_t *match; + + match = data; + + /* A negative value stops the whole match with this value. */ + + if (match->budget == 0) { + return PCRE2_ERROR_CALLOUT; + } + + match->budget--; + + return 0; +} + + +static void +nxt_pcre2_match_cleanup(nxt_task_t *task, void *obj, void *data) +{ + pcre2_match_context_free(obj); +} + + nxt_regex_match_t * nxt_regex_match_create(nxt_mp_t *mp, size_t size) { nxt_regex_match_t *match; - pcre2_general_context *ctx; + pcre2_match_context *ctx; + pcre2_general_context *general_ctx; - ctx = pcre2_general_context_create(nxt_pcre2_malloc, nxt_pcre2_free, mp); - if (nxt_slow_path(ctx == NULL)) { + match = nxt_mp_get(mp, sizeof(nxt_regex_match_t)); + if (nxt_slow_path(match == NULL)) { + return NULL; + } + + general_ctx = pcre2_general_context_create(nxt_pcre2_malloc, + nxt_pcre2_free, mp); + if (nxt_slow_path(general_ctx == NULL)) { nxt_thread_log_alert("pcre2_general_context_create() failed"); return NULL; } - match = pcre2_match_data_create(size, ctx); - if (nxt_slow_path(match == NULL)) { + match->data = pcre2_match_data_create(size, general_ctx); + if (nxt_slow_path(match->data == NULL)) { nxt_thread_log_alert("pcre2_match_data_create(%uz) failed", size); return NULL; } + /* + * PCRE2 10.30 to 10.40 allocate the backtracking frames of a match with + * the allocator of the match context, and free them after the match. + * The pool does not free memory. So the context uses the system + * allocator, and the pool frees the context when the pool is destroyed. + */ + + ctx = pcre2_match_context_create(NULL); + if (nxt_slow_path(ctx == NULL)) { + nxt_thread_log_alert("pcre2_match_context_create() failed"); + return NULL; + } + + if (nxt_slow_path(nxt_mp_cleanup(mp, nxt_pcre2_match_cleanup, NULL, ctx, + NULL) + != NXT_OK)) + { + pcre2_match_context_free(ctx); + return NULL; + } + + (void) pcre2_set_match_limit(ctx, NXT_REGEX_MATCH_LIMIT); + +#if (PCRE2_MAJOR > 10 || PCRE2_MINOR >= 30) + (void) pcre2_set_heap_limit(ctx, NXT_PCRE2_HEAP_LIMIT); +#endif + + (void) pcre2_set_callout(ctx, nxt_pcre2_callout, match); + + match->ctx = ctx; + return match; } @@ -134,13 +260,52 @@ nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, nxt_regex_match_t *match) { nxt_int_t ret; + const char *limit; PCRE2_UCHAR errptr[ERR_BUF_SIZE]; - ret = pcre2_match(re->code, (PCRE2_SPTR) subject, length, 0, 0, match, - NULL); + match->budget = NXT_REGEX_MATCH_LIMIT; + + ret = pcre2_match(re->code, (PCRE2_SPTR) subject, length, 0, 0, + match->data, match->ctx); if (nxt_slow_path(ret < PCRE2_ERROR_NOMATCH)) { + /* + * A client can cause these errors on each request. So the log gets + * no subject bytes for a limit, and at most 64 for others. The + * callout returns PCRE2_ERROR_CALLOUT when one match makes more + * than NXT_REGEX_MATCH_LIMIT callouts. + */ + + switch (ret) { + + case PCRE2_ERROR_MATCHLIMIT: + case PCRE2_ERROR_CALLOUT: + limit = "match"; + break; + +#if defined(PCRE2_ERROR_HEAPLIMIT) + case PCRE2_ERROR_HEAPLIMIT: + limit = "heap"; + break; +#endif + + default: + limit = NULL; + break; + } + + if (limit != NULL) { + nxt_thread_log_error(NXT_LOG_WARN, + "pcre2_match() reached the %s limit on %uz " + "bytes using \"%V\"", limit, length, + &re->pattern); + + return NXT_ERROR; + } + + length = nxt_min(length, NXT_REGEX_LOG_SUBJECT); + if (pcre2_get_error_message(ret, errptr, ERR_BUF_SIZE) < 0) { nxt_thread_log_error(NXT_LOG_ERR, "pcre2_match() failed: %d on \"%*s\" " diff --git a/src/nxt_regex.h b/src/nxt_regex.h index a24c50f80..70dbaa6c3 100644 --- a/src/nxt_regex.h +++ b/src/nxt_regex.h @@ -9,13 +9,24 @@ #if (NXT_HAVE_REGEX) -typedef struct nxt_regex_s nxt_regex_t; +/* + * The most match steps that one match of one pattern may use. The library + * default is 10,000,000. The library counts steps from zero again at each + * start position of a pattern that is not anchored. So such a pattern also + * gets a callout before each item, and one match may make at most this + * number of callouts in total. + */ +#define NXT_REGEX_MATCH_LIMIT 100000 - #if (NXT_HAVE_PCRE2) -typedef void nxt_regex_match_t; -#else +/* + * The most bytes of a subject that a match error writes to the log. The + * subject can come from a client. + */ +#define NXT_REGEX_LOG_SUBJECT 64 + + +typedef struct nxt_regex_s nxt_regex_t; typedef struct nxt_regex_match_s nxt_regex_match_t; -#endif typedef struct { size_t offset; diff --git a/src/test/nxt_regex_test.c b/src/test/nxt_regex_test.c new file mode 100644 index 000000000..868898878 --- /dev/null +++ b/src/test/nxt_regex_test.c @@ -0,0 +1,160 @@ +/* + * Copyright (C) FreeUnit contributors. + */ + +/* + * A pattern with two greedy groups and a class that the subject lacks + * takes about 2M match steps on 2000 separators on PCRE2 10.4x and PCRE + * 8.39. The match must stop at NXT_REGEX_MATCH_LIMIT and report an error. + * + * A repeated group takes about 2 match steps per byte. On PCRE2 it must + * still match on an 8 KiB subject. PCRE 1 also nests about 2 calls per + * byte on the stack. On an 8 KiB subject it must stop at its recursion + * limit and report an error, not overflow the stack. On a 512-byte subject + * it must match. + * + * The library counts match steps from zero again at each start position of + * a pattern that is not anchored. In PCRE2 10.42, "(?=a)(?:a|aa){0,12}[xy]" + * takes 8,193 steps and 28,659 callouts at each of the first 976 positions + * of 1000 "a" bytes and an "x". This is below the match limit at each + * position, but the whole match takes about 1 s. The callouts must stop it + * at NXT_REGEX_MATCH_LIMIT in total, with an error. The same pattern with + * "^" has one start position and must not match. + */ + +#include +#include +#include "nxt_tests.h" + + +#define NXT_REGEX_TEST_SIZE 8192 +#define NXT_REGEX_TEST_SHORT 512 +#define NXT_REGEX_TEST_STARTS 1000 + +#if (NXT_HAVE_PCRE2) +#define NXT_REGEX_TEST_LONG_RESULT 1 +#else +#define NXT_REGEX_TEST_LONG_RESULT NXT_ERROR +#endif + + +/* + * Compile the pattern and match it against the subject. The match result + * goes to *result. The function returns NXT_ERROR if the setup fails. + */ + +static nxt_int_t +nxt_regex_test_run(nxt_thread_t *thr, nxt_mp_t *mp, const char *pattern, + u_char *subject, size_t length, nxt_int_t *result) +{ + nxt_str_t source; + nxt_regex_t *re; + nxt_regex_err_t err; + nxt_regex_match_t *match; + + source.start = (u_char *) pattern; + source.length = nxt_strlen(pattern); + + re = nxt_regex_compile(mp, &source, &err); + if (re == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "regex test: compile of " + "\"%s\" failed: %s", pattern, err.msg); + return NXT_ERROR; + } + + match = nxt_regex_match_create(mp, 1); + if (match == NULL) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "regex test: match data " + "for \"%s\" failed", pattern); + return NXT_ERROR; + } + + *result = nxt_regex_match(re, subject, length, match); + + return NXT_OK; +} + + +nxt_int_t +nxt_regex_test(nxt_thread_t *thr) +{ + u_char *subject; + nxt_mp_t *mp; + nxt_int_t ret, slow, deep, shallow, starts, anchored; + + nxt_thread_time_update(thr); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (mp == NULL) { + return NXT_ERROR; + } + + subject = nxt_mp_alloc(mp, NXT_REGEX_TEST_SIZE); + if (subject == NULL) { + nxt_mp_destroy(mp); + return NXT_ERROR; + } + + subject[0] = '/'; + nxt_memset(subject + 1, '/', 2000); + + ret = nxt_regex_test_run(thr, mp, "^/(.*)/(.*)[xy]$", subject, 2001, + &slow); + + if (ret == NXT_OK) { + nxt_memset(subject + 1, 'a', NXT_REGEX_TEST_SIZE - 1); + + ret = nxt_regex_test_run(thr, mp, "^/(a|b)+$", subject, + NXT_REGEX_TEST_SIZE, &deep); + } + + if (ret == NXT_OK) { + ret = nxt_regex_test_run(thr, mp, "^/(a|b)+$", subject, + NXT_REGEX_TEST_SHORT, &shallow); + } + + if (ret == NXT_OK) { + nxt_memset(subject, 'a', NXT_REGEX_TEST_STARTS); + subject[NXT_REGEX_TEST_STARTS] = 'x'; + + ret = nxt_regex_test_run(thr, mp, "(?=a)(?:a|aa){0,12}[xy]", subject, + NXT_REGEX_TEST_STARTS + 1, &starts); + } + + if (ret == NXT_OK) { + ret = nxt_regex_test_run(thr, mp, "^(?=a)(?:a|aa){0,12}[xy]", + subject, NXT_REGEX_TEST_STARTS + 1, + &anchored); + } + + nxt_mp_destroy(mp); + + if (ret != NXT_OK) { + return NXT_ERROR; + } + + NXT_TEST_CHECK(thr->log, slow == NXT_ERROR, "regex test: backtracking " + "pattern returned %i, expected %i", slow, + (nxt_int_t) NXT_ERROR); + + NXT_TEST_CHECK(thr->log, deep == NXT_REGEX_TEST_LONG_RESULT, "regex test: " + "repeated group on %d bytes returned %i, expected %i", + NXT_REGEX_TEST_SIZE, deep, + (nxt_int_t) NXT_REGEX_TEST_LONG_RESULT); + + NXT_TEST_CHECK(thr->log, shallow == 1, "regex test: repeated group " + "on %d bytes returned %i, expected 1", + NXT_REGEX_TEST_SHORT, shallow); + + NXT_TEST_CHECK(thr->log, starts == NXT_ERROR, "regex test: pattern " + "without an anchor on %d bytes returned %i, expected %i", + NXT_REGEX_TEST_STARTS + 1, starts, (nxt_int_t) NXT_ERROR); + + NXT_TEST_CHECK(thr->log, anchored == 0, "regex test: anchored pattern " + "on %d bytes returned %i, expected 0", + NXT_REGEX_TEST_STARTS + 1, anchored); + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "regex test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 29b7b8950..5d29ce22b 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -74,6 +74,9 @@ static nxt_int_t (*const nxt_security_tests[])(nxt_thread_t *) = { nxt_checked_test, nxt_port_mmap_read_test, nxt_router_response_parse_test, nxt_port_frag_test, nxt_port_release_test, nxt_nncq_bound_test, +#if (NXT_HAVE_REGEX) + nxt_regex_test, +#endif }; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 6b6559c46..31a2c90f7 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -114,6 +114,9 @@ nxt_int_t nxt_router_response_parse_test(nxt_thread_t *thr); nxt_int_t nxt_port_frag_test(nxt_thread_t *thr); nxt_int_t nxt_port_release_test(nxt_thread_t *thr); nxt_int_t nxt_nncq_bound_test(nxt_thread_t *thr); +#if (NXT_HAVE_REGEX) +nxt_int_t nxt_regex_test(nxt_thread_t *thr); +#endif nxt_int_t nxt_openssl_server_init_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_controller_peer_test(nxt_thread_t *thr); diff --git a/test/test_routing.py b/test/test_routing.py index 0d54094a1..39ed01de5 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -297,6 +297,60 @@ def test_routes_bad_regex(require): assert client.get(url='/nothing_z')['status'] == 500, '/nothing_z' +def test_routes_regex_match_limit(require, wait_for_record, findall): + require({'modules': {'regex': True}}) + + # PCRE2 10.4x and PCRE 8.x try each split of the commas between the two + # groups, about 2M steps. They do not see that no "x" or "y" is in the + # subject. + route_match({"headers": {"x-blah": "~^(.*),(.*)[xy]$"}}) + + assert ( + client.get(headers={'Host': 'localhost', 'X-Blah': ',' * 2000, + 'Connection': 'close'})['status'] + == 500 + ), 'match limit' + + assert ( + wait_for_record(r'\[warn\].+reached the match limit on 2000 bytes') + is not None + ), 'match limit log' + assert not findall(r',{64}'), 'no subject in the log' + + assert ( + client.get(headers={'Host': 'localhost', 'X-Blah': 'a,bx', + 'Connection': 'close'})['status'] + == 200 + ), 'short subject' + + +def test_routes_regex_match_limit_unanchored(require, wait_for_record): + require({'modules': {'regex': True}}) + + # The library counts the steps from zero again at each start position. + # This pattern takes about 8,000 steps at each of the first 976 + # positions, below the limit at each one. The limit must stop the + # whole match. + route_match({"headers": {"x-blah": "~(?=a)(?:a|aa){0,12}[xy]"}}) + + assert ( + client.get(headers={'Host': 'localhost', 'X-Blah': 'a' * 1000 + 'x', + 'Connection': 'close'})['status'] + == 500 + ), 'match limit' + + assert ( + wait_for_record(r'\[warn\].+reached the match limit on 1001 bytes') + is not None + ), 'match limit log' + + assert ( + client.get(headers={'Host': 'localhost', 'X-Blah': 'aax', + 'Connection': 'close'})['status'] + == 200 + ), 'short subject' + + def test_routes_match_regex_captures(require): require({'modules': {'regex': True}}) diff --git a/test/test_static_compression.py b/test/test_static_compression.py index ac0bbecfc..62d7dc8b6 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -52,6 +52,66 @@ def test_static_compression_baseline(): assert resp['headers']['Content-Encoding'] == 'gzip', 'gzip applied' +def test_static_compression_regex_match_limit( + require, temp_dir, wait_for_record +): + # A "types" regex that reaches its match limit is not a match. So the + # response is not compressed. + require({'modules': {'regex': True}}) + + assets_dir = f'{temp_dir}/assets' + Path(f'{assets_dir}/a.slow').write_text('slow' * 10, encoding='utf-8') + Path(f'{assets_dir}/a.fast').write_text('fast' * 10, encoding='utf-8') + + assert 'success' in client.conf( + { + "static": { + "mime_types": {',' * 2000: ".slow", "a,bx": ".fast"} + }, + "compression": { + "types": ["~^(.*),(.*)[xy]$"], + "compressors": [ + {"encoding": "gzip", "level": 5, "min_length": 10} + ], + }, + }, + 'settings/http', + ), 'regex types configure' + + headers = { + 'Host': 'localhost', + 'Accept-Encoding': 'gzip', + 'Connection': 'close', + } + + resp = client.get(url='/a.fast', headers=headers) + assert resp['status'] == 200, 'fast status' + assert resp['headers'].get('Content-Encoding') == 'gzip', 'fast gzip' + + # gzip is selected, but the "types" regex reaches its match limit, and + # that is not a match. So a client that refuses identity gets a 406, + # not the bytes that it refused. + resp = client.get( + url='/a.slow', + headers={ + 'Host': 'localhost', + 'Accept-Encoding': 'gzip, identity;q=0', + 'Connection': 'close', + }, + ) + assert resp['status'] == 406, 'slow identity refused' + + resp = client.get(url='/a.slow', headers=headers) + assert resp['status'] == 200, 'slow status' + assert 'Content-Encoding' not in resp['headers'], 'slow identity' + assert resp['body'] == 'slow' * 10, 'slow body' + + assert ( + wait_for_record(r'\[warn\].+reached the match limit on 2000 bytes') + is not None + ), 'match limit log' + + def test_static_compression_removed_between_requests(): # #167: the compression state used to be process-global and allocated # from the router configuration that parsed it, so a configuration From f69822515a3f8f777e17a5514462e4db5451582d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 27 Sep 2026 14:54:21 +0000 Subject: [PATCH 338/391] http: add body_min_rate and send_min_rate The HTTP/1 body_read_timeout and send_timeout timers are gap timers. Each read or write of one or more bytes starts the timer again. Thus a client that sends or reads one byte just before each timeout keeps a connection for ever ("slow POST", "slow read"). On master: - body_read_timeout 2, Content-Length 10000, 1 byte each 1.5 s: the connection is still open after 18.6 s. - send_timeout 2, 64 MiB static file, the client reads 1 KiB each 1.5 s: the connection is still open after 21 s. The router keeps the full body before it sends it to the application. max_body_size limits only the bytes, not the time. This change adds two options to "settings/http": - "body_min_rate": the minimum client body rate in bytes per second. - "send_min_rate": the minimum rate at which the client reads the response, in bytes per second. The value is an integer from 0 to 2147483647. The default is 0, which disables the check. Thus the change does not change the behavior of current configurations. For body_min_rate, 256 is a good value. send_min_rate counts the bytes that the kernel accepts into the socket send buffer, which can hold some MiB. A reader slower than a small floor then gets no write events, and send_timeout stops it first. On a real network the kernel sizes this buffer to the link, so the measured rate is near the real download rate of the client. A floor above the bandwidth of honest slow clients stops their downloads. Thus the docs advise a small value. docs/unit-openapi.yaml uses 256 as the body_min_rate example and 16384 as the send_min_rate example, in the endpoint examples and in the settings examples. When the related timeout is 0, there is no grace time and each read or write is checked. The docs say this. The check starts after a grace time. The grace time is equal to the related timeout (body_read_timeout or send_timeout). It lets TCP slow start and short network stops occur. After the grace time, when a read or a write completes, the router compares "bytes * 1000" with "rate * elapsed ms" in 64-bit integers. - Body: the router counts each byte that a read returns in the body read state, chunk framing included. The framing costs the client the same bandwidth, and max_body_size limits the payload. The header block, and the body bytes that came with it, are not counted. The router adds the bytes only when body_min_rate is set. The time starts at the start of the body read state. With "Expect: 100-continue", that state starts when the router has sent the 100, so the time starts again there. The body_read_timeout timer also starts there. Thus the time when the 100 waits for space in the send buffer is not counted. On failure the router uses the body_read_timeout steps: 408, no keep-alive, close. The read that completes the body is not checked. It is the last body event, so a 408 there frees the connection no earlier: it only refuses a body that is complete. The reads before it are checked, and it must come within body_read_timeout of the read before it. Apache mod_reqtimeout MinRate also never refuses a body that is complete. - Send: the router counts the bytes that the kernel accepts. It measures only the time when response data waits for the client. A period starts when data goes to an empty write queue, and stops when the queue is empty again. Thus a slow application or upstream (for example a stream of events) is not stopped. The check runs after each write, also after the write that empties the queue. An application or upstream that gives one buffer at a time can make each write event empty the queue. Without the check there, a client that keeps the socket buffer full is never checked. On failure the router uses the send_timeout steps: the connection closes. When the last data of the response goes out, the response is complete in the socket buffer, so a failure there only stops keep-alive. Both failures write an info message to the log. The rate is checked for each window. When a check passes and the window is at least the grace time long, the bytes and the time start again from zero. Without this, bytes sent early give credit for a slow transfer later: 50000 bytes at once and then 1 byte each 1.5 s kept a body open for about 195 s at 256 bytes per second. With it, the first slow window after the burst gives 408. The floor is added to the gap timers. It does not replace them. The floor check runs only on a read or write event. A client that stops fully causes no events, and the gap timer stops it. A client that is slower than the floor still causes events, and the check stops it. A total time limit is not used, because it also stops honest large uploads and downloads on slow links. A rate floor stops only clients that are slower than the floor. The state is in nxt_h1proto_t before the "conn" field, so the keep-alive code zeroes it for each request. Cost: nxt_h1proto_t grows from 648 to 696 bytes, 48 bytes for each connection (a probe compiled against the headers of ac2f2d29 and of this change). The keep-alive memzero clears 48 bytes more for each request. With both options 0, a body read costs one flag test. A response costs one call that returns at once, and each write completion one call that returns after one flag test. With an option set, a read or a write completion adds a few integer operations. No path adds an allocation, a system call or a lock. Scope: only the client HTTP/1 request body read and response send. The proxy peer states, WebSocket frames and the TLS handshake do not use the floor. Tests in test/test_settings.py: - test_settings_body_min_rate_slow_body: 1 byte each second with body_read_timeout 2 and body_min_rate 256 is stopped in less than 8 s. It fails when the check is disabled. - test_settings_body_min_rate_burst_then_slow: 50000 bytes at once, then 1 byte each second, is stopped in less than 12 s (4 s here). It fails without the window reset. - test_settings_body_min_rate_normal_body: a fast body gets 200, also for a second request on a keep-alive connection after an idle time. - test_settings_body_min_rate_expect_slow_body: with "Expect: 100-continue", the client reads the 100 and then sends 1 byte each second. It is stopped in less than 8 s. It fails when the floor is set up only on the path without a 100. - test_settings_body_min_rate_expect_wait: the client reads the 100, waits 1 s, and then sends 20000 bytes in 2 s. It gets 200. It fails when nxt_h1p_conn_continue_sent() sets the start time to 0. No test shows the new start of the time itself. On a new connection the 100 is sent in the same event loop turn as the first start, so the time value is the same. - test_settings_send_min_rate_slow_read: a client that reads slower than the floor, but reads enough for a write before each send_timeout, is stopped. On loopback the kernel buffers are some MiB, so the test uses a floor of 8 MiB/s and reads about 1 MiB/s. It fails when the check is disabled. It runs only on Linux, because the numbers are for the Linux loopback buffers. - test_settings_send_min_rate_slow_application: the application sends 4 parts with 1 s between them (send_timeout 2, floor 8 MiB/s). The client gets the full body. It fails when the floor also counts the time when the router waits for the application: with a period that does not stop when the queue is empty, the client gets 3000 of 4000 bytes. - test_settings_send_min_rate_gap_longer_than_grace: the application sends 1000 bytes, waits 2.5 s, and sends 1000 bytes more (send_timeout 2, floor 1 MiB/s). The client gets the full body, and a second request on the same connection gets 200. It fails when a new send period does not start its own time, so that the gap is counted: the log has "2099 bytes in 2500 ms", and the second request finds the connection closed ("assert b''"). - test_settings_send_min_rate_stream_slow_read: over a unix listener, the application sends 32 KiB parts in bursts 1.5 s apart. The client keeps the socket buffer full and reads one burst each 1.5 s, so each write event empties the write queue. The client must be stopped. Without the check when the queue empties, it gets the full stream: "assert 1081344 < 1081344". On Linux, the kernel takes a unix socket write of 32 KiB whole or not at all, so no write leaves data in the queue. It runs only on Linux. - test_settings_send_min_rate_fast_read: a fast reader gets all data, also on a keep-alive connection. - test_settings_min_rate_validation: -1, 1.5, "256" and 2^31 are refused; 0, 256 and 2^31 - 1 are accepted. The current body_read_timeout and send_timeout tests pass without change. Co-Authored-By: Claude Opus 5.5 Reviewed-by: DeepSeek V4.1 Flash Harness: DeepSeek Harness Reviewed-by: Claude Fable 5.1 Reviewed-by: OpenAI Codex --- CHANGES | 16 + docs/changes.xml | 18 + docs/unit-openapi.yaml | 210 ++++++++++ src/nxt_conf_validation.c | 42 ++ src/nxt_h1proto.c | 248 +++++++++++- src/nxt_h1proto.h | 13 + src/nxt_router.c | 14 + src/nxt_router.h | 4 + test/python/send_bursts/wsgi.py | 31 ++ test/test_settings.py | 684 ++++++++++++++++++++++++++++++++ 10 files changed, 1276 insertions(+), 4 deletions(-) create mode 100644 test/python/send_bursts/wsgi.py diff --git a/CHANGES b/CHANGES index 38dc36100..e172962d7 100644 --- a/CHANGES +++ b/CHANGES @@ -201,6 +201,22 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 such a response gave 502 when it carried chunked together with Content-Length. + *) Feature: the "body_min_rate" and "send_min_rate" options in + "settings/http" set a minimum client rate in bytes per second. + The body_read_timeout and send_timeout timers start again after + each read or write, so a client that sends or reads one byte + before each timeout kept a connection for ever. After a grace + time equal to the related timeout, a client below the rate gets + 408 (body) or the connection closes (response). The rate is + checked for each window of at least the grace time, so bytes sent + early give no credit for a slow transfer later. The default is 0, + which disables the check. For body_min_rate, 256 is a good value. + send_min_rate counts the bytes that the kernel accepts into the + socket send buffer. The kernel sizes this buffer to the link, so + the rate is near the real download rate of the client. A floor + above the bandwidth of honest slow clients stops their downloads; + use a small value, for example 16384. + *) Feature: the --hardening=[off|default|strict] configure option adds compiler and linker hardening flags. Each flag is probed, and a compiler that does not support a flag only loses that flag. The diff --git a/docs/changes.xml b/docs/changes.xml index 2433b95cc..ef20b074c 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -298,6 +298,24 @@ it carried chunked together with Content-Length. + + +the "body_min_rate" and "send_min_rate" options in "settings/http" set +a minimum client rate in bytes per second. The body_read_timeout and +send_timeout timers start again after each read or write, so a client that +sends or reads one byte before each timeout kept a connection for ever. +After a grace time equal to the related timeout, a client below the rate +gets 408 (body) or the connection closes (response). The rate is checked for +each window of at least the grace time, so bytes sent early give no credit for +a slow transfer later. The default is 0, which disables the check. For +body_min_rate, 256 is a good value. send_min_rate counts the bytes that the +kernel accepts into the socket send buffer. The kernel sizes this buffer to +the link, so the rate is near the real download rate of the client. A floor +above the bandwidth of honest slow clients stops their downloads; use a small +value, for example 16384. + + + the --hardening=[off|default|strict] configure option adds compiler and diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index fa25cb7e2..94f00a08a 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -3285,6 +3285,87 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /config/settings/http/body_min_rate: + summary: "Endpoint for the `body_min_rate` option in `http`" + get: + operationId: getSettingsHttpBodyMinRate + summary: "Retrieve the body_min_rate option from http settings" + description: "Retrieves the `body_min_rate` option that represents + the minimum rate of the client request body, in bytes per second. + The value 0 disables the check." + + tags: + - settings + - config + + responses: + "200": + description: "OK; the `body_min_rate` option exists in the + configuration." + + content: + application/json: + schema: + type: integer + + examples: + BodyMinRate: + value: 256 + + "404": + $ref: "#/components/responses/responseNotFound" + + put: + operationId: updateSettingsHttpBodyMinRate + summary: "Create or overwrite the body_min_rate option" + description: "Creates or overwrites the `body_min_rate` option in + the configuration." + + tags: + - settings + - config + + requestBody: + required: true + content: + application/json: + schema: + type: integer + + examples: + BodyMinRate: + value: 256 + + responses: + "200": + $ref: "#/components/responses/responseOkUpdated" + + "400": + $ref: "#/components/responses/responseBadRequest" + + "404": + $ref: "#/components/responses/responseNotFound" + + "500": + $ref: "#/components/responses/responseInternalError" + + delete: + operationId: deleteSettingsHttpBodyMinRate + summary: "Delete the body_min_rate option" + description: "Deletes the `body_min_rate` option from the + configuration." + + tags: + - settings + - config + + responses: + "200": + $ref: "#/components/responses/responseOkDeleted" + + "404": + $ref: "#/components/responses/responseNotFound" + /config/settings/http/body_read_timeout: summary: "Endpoint for the `body_read_timeout` option in `http`" get: @@ -4525,6 +4606,87 @@ paths: "404": $ref: "#/components/responses/responseNotFound" + /config/settings/http/send_min_rate: + summary: "Endpoint for the `send_min_rate` option in `http`" + get: + operationId: getSettingsHttpSendMinRate + summary: "Retrieve the send_min_rate option from http settings" + description: "Retrieves the `send_min_rate` option that represents + the minimum rate at which the client reads the response, in bytes + per second. The value 0 disables the check." + + tags: + - settings + - config + + responses: + "200": + description: "OK; the `send_min_rate` option exists in the + configuration." + + content: + application/json: + schema: + type: integer + + examples: + SendMinRate: + value: 16384 + + "404": + $ref: "#/components/responses/responseNotFound" + + put: + operationId: updateSettingsHttpSendMinRate + summary: "Create or overwrite the send_min_rate option" + description: "Creates or overwrites the `send_min_rate` option in + the configuration." + + tags: + - settings + - config + + requestBody: + required: true + content: + application/json: + schema: + type: integer + + examples: + SendMinRate: + value: 16384 + + responses: + "200": + $ref: "#/components/responses/responseOkUpdated" + + "400": + $ref: "#/components/responses/responseBadRequest" + + "404": + $ref: "#/components/responses/responseNotFound" + + "500": + $ref: "#/components/responses/responseInternalError" + + delete: + operationId: deleteSettingsHttpSendMinRate + summary: "Delete the send_min_rate option" + description: "Deletes the `send_min_rate` option from the + configuration." + + tags: + - settings + - config + + responses: + "200": + $ref: "#/components/responses/responseOkDeleted" + + "404": + $ref: "#/components/responses/responseNotFound" + /config/settings/http/send_timeout: summary: "Endpoint for the `send_timeout` option in `http`" get: @@ -6019,12 +6181,14 @@ components: settings: http: + body_min_rate: 256 body_read_timeout: 30 discard_unsafe_fields: true header_read_timeout: 30 idle_timeout: 180 log_route: true max_body_size: 8388608 + send_min_rate: 16384 send_timeout: 30 server_version: false @@ -6291,6 +6455,7 @@ components: summary: "Global settings" value: http: + body_min_rate: 256 body_read_timeout: 30 compression: types: @@ -6305,6 +6470,7 @@ components: idle_timeout: 180 log_route: true max_body_size: 8388608 + send_min_rate: 16384 send_timeout: 30 server_version: false static: @@ -6317,6 +6483,7 @@ components: configSettingsHttp: summary: "HTTP settings" value: + body_min_rate: 256 body_read_timeout: 30 compression: types: @@ -6331,6 +6498,7 @@ components: idle_timeout: 180 log_route: true max_body_size: 8388608 + send_min_rate: 16384 send_timeout: 30 server_version: false static: @@ -7873,6 +8041,26 @@ components: in Unit." properties: + body_min_rate: + type: integer + minimum: 0 + maximum: 2147483647 + description: "Minimum rate of the client request body, in bytes + per second. The check starts after a grace time. The grace + time is equal to `body_read_timeout`. After the grace time, + if the client sent fewer bytes than `body_min_rate` multiplied + by the elapsed seconds, Unit sends 408 and closes the + connection. The rate is checked for each window of at least + the grace time, so bytes sent early give no credit later. Unit + counts the body bytes that it reads after the request header. + The read that completes the body is not checked, because the + body is then complete. If `body_read_timeout` is 0, there is + no grace time and each read is checked. The value 0 disables + the check. Use the + value 256 to stop slow clients (\"slow POST\")." + + default: 0 + body_read_timeout: type: integer description: "Maximum number of seconds to read data from the body of @@ -7917,6 +8105,28 @@ components: default: 8388608 + send_min_rate: + type: integer + minimum: 0 + maximum: 2147483647 + description: "Minimum rate at which the client reads the response, + in bytes per second. Unit measures only the time when response + data waits for the client. The check starts after a grace time. + The grace time is equal to `send_timeout`. After the grace + time, if the client accepted fewer bytes than `send_min_rate` + multiplied by the elapsed seconds, Unit closes the connection. + The rate is checked for each window of at least the grace time. + Unit counts the bytes that the kernel accepts into the socket + send buffer. The kernel sizes this buffer to the link, so the + measured rate is near the real download rate of the client. A + floor above the bandwidth of honest slow clients stops their + downloads. Use a small value, for example 16384. If + `send_timeout` is 0, there is no grace time and each write is + checked. The value 0 disables the check. WebSocket + connections do not use this check." + + default: 0 + send_timeout: type: integer description: "Maximum number of seconds to transmit data as a diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index d16953028..67579d2cf 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -143,6 +143,8 @@ static nxt_int_t nxt_conf_vldt_python_prefix(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_listen_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_min_rate(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, @@ -395,6 +397,16 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("send_timeout"), .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("body_min_rate"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_min_rate, + .u.string = "body_min_rate", + }, { + .name = nxt_string("send_min_rate"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_min_rate, + .u.string = "send_min_rate", }, { .name = nxt_string("idle_timeout"), .type = NXT_CONF_VLDT_INTEGER, @@ -2797,6 +2809,36 @@ nxt_conf_vldt_listen_threads(nxt_conf_validation_t *vldt, } +/* + * The rate is in bytes per second. The router compares it with the + * elapsed time in milliseconds as a 64-bit product. The upper limit + * keeps this product in range. + */ + +static nxt_int_t +nxt_conf_vldt_min_rate(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + int64_t rate; + const char *name; + + name = data; + rate = nxt_conf_get_number(value); + + if (rate < 0) { + return nxt_conf_vldt_error(vldt, "The \"%s\" number must be " + "equal to or greater than 0.", name); + } + + if (rate > NXT_INT32_T_MAX) { + return nxt_conf_vldt_error(vldt, "The \"%s\" number must " + "not exceed %d.", name, NXT_INT32_T_MAX); + } + + return NXT_OK; +} + + static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 66179255e..febe9f0e0 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -64,6 +64,12 @@ static nxt_off_t nxt_h1p_request_body_bytes_sent(nxt_task_t *task, static void nxt_h1p_request_discard(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *last); static void nxt_h1p_conn_request_error(nxt_task_t *task, void *obj, void *data); +static nxt_bool_t nxt_h1p_rate_too_low(uint64_t bytes, uint64_t msec, + nxt_msec_t grace, int32_t rate); +static void nxt_h1p_send_rate_start(nxt_task_t *task, nxt_h1proto_t *h1p, + nxt_http_request_t *r); +static nxt_bool_t nxt_h1p_send_rate_check(nxt_task_t *task, nxt_conn_t *c); +static void nxt_h1p_request_timedout(nxt_task_t *task, nxt_conn_t *c); static void nxt_h1p_conn_request_timeout(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_conn_request_send_timeout(nxt_task_t *task, void *obj, @@ -1103,6 +1109,21 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) h1p->buffers = in; h1p->nbuffers++; + /* + * The body_min_rate floor counts each byte that a read returns + * in the body read state, chunk framing included. The framing + * costs the client the same bandwidth, and max_body_size limits + * the payload. The header block, and the body bytes that came + * with it, are not counted, because the first body read comes + * after this point. Thus the first window is a little stricter. + * This is intentional. The time starts at the start of the body + * read state. After a 100 (Continue), + * nxt_h1p_conn_continue_sent() starts it again. + */ + h1p->body_rate_on = (skcf->body_min_rate > 0); + h1p->body_rate_start = task->thread->engine->timers.now; + h1p->body_rate_bytes = 0; + c = h1p->conn; c->read = b; @@ -1207,6 +1228,7 @@ static void nxt_h1p_conn_continue_sent(nxt_task_t *task, void *obj, void *data) { nxt_conn_t *c; + nxt_h1proto_t *h1p; nxt_event_engine_t *engine; c = obj; @@ -1222,6 +1244,14 @@ nxt_h1p_conn_continue_sent(nxt_task_t *task, void *obj, void *data) return; } + /* + * The body read state starts now, and so does the body_read_timeout + * timer. Thus the body_min_rate time starts again here. The time + * when the 100 waited for space in the send buffer is not counted. + */ + h1p = c->socket.data; + h1p->body_rate_start = engine->timers.now; + c->read_state = &nxt_h1p_read_body_state; nxt_conn_read(engine, c); @@ -1248,6 +1278,7 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) size_t size, body_rest; ssize_t res; nxt_buf_t *b, *out, *chunk; + nxt_msec_t msec; nxt_conn_t *c; nxt_h1proto_t *h1p; nxt_socket_conf_t *skcf; @@ -1362,6 +1393,36 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "h1p body rest: %uz", body_rest); if (body_rest != 0) { + + if (h1p->body_rate_on) { + /* + * The bytes are counted only here. The flag does not change + * in the body read state, and only a read that continues + * the body is checked. + */ + h1p->body_rate_bytes += c->nbytes; + + msec = (nxt_msec_t) (engine->timers.now - h1p->body_rate_start); + + if (nxt_h1p_rate_too_low(h1p->body_rate_bytes, msec, + skcf->body_read_timeout, + skcf->body_min_rate)) + { + nxt_log(task, NXT_LOG_INFO, "client body rate is less than " + "body_min_rate %d: %uL bytes in %M ms", + skcf->body_min_rate, h1p->body_rate_bytes, msec); + + nxt_h1p_request_timedout(task, c); + return; + } + + if (msec >= skcf->body_read_timeout) { + /* The window passed: the next window counts from now. */ + h1p->body_rate_start = engine->timers.now; + h1p->body_rate_bytes = 0; + } + } + nxt_conn_read(engine, c); } else { @@ -1680,6 +1741,8 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, h1p->conn_write_tail = &header->next; c->write_state = &nxt_h1p_request_send_state; + nxt_h1p_send_rate_start(task, h1p, r); + if (body_handler != NULL) { /* * The body handler will run before c->io->write() handler, @@ -1785,6 +1848,8 @@ nxt_h1p_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) c->write = out; c->write_state = &nxt_h1p_request_send_state; + nxt_h1p_send_rate_start(task, h1p, r); + nxt_conn_write(task->thread->engine, c); } else { @@ -1959,19 +2024,188 @@ nxt_h1p_conn_request_error(nxt_task_t *task, void *obj, void *data) } +/* + * The minimum transfer rate floor: body_min_rate and send_min_rate. + * + * The body_read_timeout and send_timeout timers are gap timers. Each + * read or write of one or more bytes starts the timer again. Thus the + * timers stop a client that sends or reads nothing, but they do not stop + * a client that transfers one byte just before each timeout ("slow POST" + * and "slow read"). Such a client can keep a connection for ever. + * + * The rate floor stops this client. The floor is added to the gap timers, + * it does not replace them. The floor check runs only when a read or + * a write completes. A client that stops fully causes no events, and the + * gap timer stops it. A client that is slower than the floor still causes + * events, and the floor check stops it. + * + * The check starts after a grace time. The grace time is equal to the gap + * timeout (body_read_timeout or send_timeout). It lets TCP slow start and + * short network stops occur. A total time limit is not used, because it + * also stops honest large transfers on slow links. + * + * The check is "bytes * 1000 < rate * msec" in 64-bit integers. The + * validator keeps the rate at or below 2^31 - 1, and the check limits + * msec to 2^31 - 1. Thus the product cannot overflow. + */ + +static nxt_bool_t +nxt_h1p_rate_too_low(uint64_t bytes, uint64_t msec, nxt_msec_t grace, + int32_t rate) +{ + if (rate <= 0 || msec < grace) { + return 0; + } + + msec = nxt_min(msec, (uint64_t) NXT_INT32_T_MAX); + + if (bytes >= UINT64_MAX / 1000) { + return 0; + } + + return (bytes * 1000 < (uint64_t) rate * msec); +} + + +/* + * The send rate is measured only while response data waits for the client. + * A send period starts when the router gives data to an empty connection + * write queue. The period stops when the client accepted all the queued + * data. The time between periods is not counted: then the router waits + * for the application or for the upstream server, not for the client. + * Thus a slow response source (for example a stream of events) is not + * stopped by the floor. The bytes and the time of the periods of one + * request are added together until a window of at least the grace time + * passes the check; then the next window starts from zero. Thus bytes + * sent early in a response give no credit for a slow read later. + * + * The check runs after each write, also after the write that stops + * a period. When the last data of the response goes out, the response + * is complete in the socket buffer. Then a failed check only closes the + * connection instead of keeping it alive. + * + * WebSocket frames also use the request send state. The floor is not + * used for a WebSocket connection. + */ + static void -nxt_h1p_conn_request_timeout(nxt_task_t *task, void *obj, void *data) +nxt_h1p_send_rate_start(nxt_task_t *task, nxt_h1proto_t *h1p, + nxt_http_request_t *r) { - nxt_conn_t *c; - nxt_timer_t *timer; + if (h1p->send_rate_on + || h1p->websocket + || r->conf->socket_conf->send_min_rate <= 0) + { + return; + } + + h1p->send_rate_on = 1; + h1p->send_rate_start = task->thread->engine->timers.now; + h1p->send_rate_sent = h1p->conn->sent; +} + + +static nxt_bool_t +nxt_h1p_send_rate_check(nxt_task_t *task, nxt_conn_t *c) +{ + uint64_t bytes, msec; nxt_h1proto_t *h1p; + nxt_socket_conf_t *skcf; nxt_http_request_t *r; + h1p = c->socket.data; + + if (!h1p->send_rate_on) { + return 0; + } + + r = h1p->request; + + if (nxt_slow_path(r == NULL)) { + return 0; + } + + skcf = r->conf->socket_conf; + + msec = h1p->send_rate_time + + (nxt_msec_t) (task->thread->engine->timers.now + - h1p->send_rate_start); + bytes = h1p->send_rate_bytes; + + if (c->sent > h1p->send_rate_sent) { + bytes += c->sent - h1p->send_rate_sent; + } + + /* + * The check also runs when the write empties the queue. An application + * or an upstream server that gives one buffer at a time can make each + * write event empty the queue. Without the check there, the router + * never checks the rate of a client that keeps the socket buffer full. + */ + + if (!nxt_h1p_rate_too_low(bytes, msec, skcf->send_timeout, + skcf->send_min_rate)) + { + if (msec >= skcf->send_timeout) { + /* The window passed: the next window counts from now. */ + msec = 0; + bytes = 0; + + h1p->send_rate_time = 0; + h1p->send_rate_bytes = 0; + h1p->send_rate_start = task->thread->engine->timers.now; + h1p->send_rate_sent = c->sent; + } + + if (c->write == NULL) { + /* The client accepted all the queued data: the period stops. */ + h1p->send_rate_on = 0; + h1p->send_rate_time = msec; + h1p->send_rate_bytes = bytes; + } + + return 0; + } + + nxt_log(task, NXT_LOG_INFO, "client send rate is less than " + "send_min_rate %d: %uL bytes in %uL ms", + skcf->send_min_rate, bytes, msec); + + /* The same steps as nxt_h1p_conn_request_send_timeout(). */ + + nxt_timer_disable(task->thread->engine, &c->write_timer); + c->block_write = 1; + + nxt_h1p_request_error(task, h1p, r); + + return 1; +} + + +static void +nxt_h1p_conn_request_timeout(nxt_task_t *task, void *obj, void *data) +{ + nxt_timer_t *timer; + timer = obj; nxt_debug(task, "h1p conn request timeout"); - c = nxt_read_timer_conn(timer); + nxt_h1p_request_timedout(task, nxt_read_timer_conn(timer)); +} + + +/* + * The steps for a request read timeout: header_read_timeout, + * body_read_timeout, and the body_min_rate floor. + */ + +static void +nxt_h1p_request_timedout(nxt_task_t *task, nxt_conn_t *c) +{ + nxt_h1proto_t *h1p; + nxt_http_request_t *r; + c->block_read = 1; /* * Disable SO_LINGER off during socket closing @@ -2083,6 +2317,12 @@ nxt_h1p_conn_sent(nxt_task_t *task, void *obj, void *data) c->write = nxt_sendbuf_completion(task, &engine->fast_work_queue, c->write); + if (c->write_state == &nxt_h1p_request_send_state + && nxt_slow_path(nxt_h1p_send_rate_check(task, c))) + { + return; + } + if (c->write != NULL) { nxt_conn_write(engine, c); } diff --git a/src/nxt_h1proto.h b/src/nxt_h1proto.h index 1137db99e..7d16142a9 100644 --- a/src/nxt_h1proto.h +++ b/src/nxt_h1proto.h @@ -57,6 +57,19 @@ struct nxt_h1proto_s { */ uint32_t sent_before_body; + /* + * The minimum transfer rate state of the current request. The + * keep-alive code zeroes these fields before the next request. + */ + uint8_t body_rate_on; /* 1 bit */ + uint8_t send_rate_on; /* 1 bit */ + nxt_msec_t body_rate_start; + nxt_msec_t send_rate_start; + uint64_t body_rate_bytes; + uint64_t send_rate_bytes; + uint64_t send_rate_time; + nxt_off_t send_rate_sent; + nxt_http_field_t *websocket_key; nxt_h1p_websocket_timer_t *websocket_timer; diff --git a/src/nxt_router.c b/src/nxt_router.c index a8c809c40..e00bc6dc9 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -2833,6 +2833,18 @@ static nxt_conf_map_t nxt_router_http_conf[] = { offsetof(nxt_socket_conf_t, send_timeout), }, + { + nxt_string("body_min_rate"), + NXT_CONF_MAP_INT32, + offsetof(nxt_socket_conf_t, body_min_rate), + }, + + { + nxt_string("send_min_rate"), + NXT_CONF_MAP_INT32, + offsetof(nxt_socket_conf_t, send_min_rate), + }, + { nxt_string("body_temp_path"), NXT_CONF_MAP_STR, @@ -3417,6 +3429,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, skcf->proxy_timeout = 60 * 1000; skcf->proxy_send_timeout = 30 * 1000; skcf->proxy_read_timeout = 30 * 1000; + skcf->body_min_rate = 0; + skcf->send_min_rate = 0; skcf->server_version = 1; skcf->chunked_transform = 0; diff --git a/src/nxt_router.h b/src/nxt_router.h index d886dad1f..fbe7b27f9 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -289,6 +289,10 @@ typedef struct { nxt_msec_t proxy_send_timeout; nxt_msec_t proxy_read_timeout; + /* Minimum client transfer rates in bytes per second, 0 is off. */ + int32_t body_min_rate; + int32_t send_min_rate; + nxt_websocket_conf_t websocket_conf; nxt_str_t body_temp_path; diff --git a/test/python/send_bursts/wsgi.py b/test/python/send_bursts/wsgi.py new file mode 100644 index 000000000..8fd23960b --- /dev/null +++ b/test/python/send_bursts/wsgi.py @@ -0,0 +1,31 @@ +import time + + +def application(environ, start_response): + # Sends the body in bursts of parts. The first burst has X-First parts, + # each next burst has X-Parts parts. A burst starts each X-Interval + # seconds. The parts of a burst are 10 ms apart, so the router gets and + # writes each part alone. + first = int(environ.get('HTTP_X_FIRST', 1)) + parts = int(environ.get('HTTP_X_PARTS', 1)) + size = int(environ.get('HTTP_X_PART_SIZE', 1)) + bursts = int(environ.get('HTTP_X_BURSTS', 1)) + interval = float(environ.get('HTTP_X_INTERVAL', 0)) + + length = size * (first + parts * (bursts - 1)) + + write = start_response('200', [('Content-Length', str(length))]) + + part = b'x' * size + start = time.monotonic() + + for burst in range(bursts): + delay = start + burst * interval - time.monotonic() + if delay > 0: + time.sleep(delay) + + for _ in range(first if burst == 0 else parts): + write(part) + time.sleep(0.01) + + return [] diff --git a/test/test_settings.py b/test/test_settings.py index 64e86150b..a37ad8359 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -375,6 +375,690 @@ def req(addr, data_len): client.conf_delete('settings/http/send_timeout') +def min_rate_conf(conf): + assert 'success' in client.conf({'http': conf}, 'settings') + + +def min_rate_reset(): + # The default no-restart suite preserves /settings between tests. + for name in ( + 'body_read_timeout', + 'body_min_rate', + 'send_timeout', + 'send_min_rate', + ): + client.conf_delete(f'settings/http/{name}') + + +def test_settings_body_min_rate_slow_body(): + # A client sends the body 1 byte each second. Each byte starts the + # body_read_timeout gap timer again, so only the rate floor can stop + # this client. The floor check starts after the grace time (equal to + # body_read_timeout, 2 s). The first body read after the grace time + # must give 408. + client.load('empty') + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + try: + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(1) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Content-Length: 10000\r\n' + b'Connection: close\r\n\r\n' + ) + + start = time.monotonic() + data = b'' + closed = False + + while time.monotonic() - start < 12: + try: + sock.sendall(b'x') + except OSError: + closed = True + break + + try: + chunk = sock.recv(4096) + except socket.timeout: + continue + except OSError: + closed = True + break + + if not chunk: + closed = True + break + + data += chunk + + elapsed = time.monotonic() - start + sock.close() + + # The router closes the connection after the 408 response. The + # client can get a reset before it reads the response, because + # the router does not read the rest of the body. + assert closed or data, 'slow body stopped' + assert data == b'' or data.startswith(b'HTTP/1.1 408'), 'slow body 408' + assert elapsed < 8, 'slow body stopped in time' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_burst_then_slow(): + # A client sends half of the body at once, then 1 byte each second. + # With one average over the whole body, the burst would give credit + # for about 195 s at 256 bytes per second. The rate is checked for + # each window of at least body_read_timeout, so the first slow window + # must give 408. + client.load('empty') + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + try: + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(1) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Content-Length: 100000\r\n' + b'Connection: close\r\n\r\n' + ) + sock.sendall(b'x' * 50000) + + start = time.monotonic() + data = b'' + closed = False + + while time.monotonic() - start < 20: + try: + sock.sendall(b'x') + except OSError: + closed = True + break + + try: + chunk = sock.recv(4096) + except socket.timeout: + continue + except OSError: + closed = True + break + + if not chunk: + closed = True + break + + data += chunk + + elapsed = time.monotonic() - start + sock.close() + + assert closed or data, 'burst then slow body stopped' + assert data == b'' or data.startswith( + b'HTTP/1.1 408' + ), 'burst then slow body 408' + assert elapsed < 12, 'burst then slow body stopped in time' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_normal_body(): + # A body that is sent faster than the floor must not be stopped. + # The second request on the same keep-alive connection comes after + # an idle time that is longer than the grace time. It must start + # with new rate state. + client.load('empty') + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + def req(sock, close): + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Content-Length: 100000\r\n' + + (b'Connection: close\r\n' if close else b'') + + b'\r\n' + ) + + for _ in range(10): + time.sleep(0.2) + sock.sendall(b'x' * 10000) + + return sock.recv(4096) + + try: + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(10) + + assert req(sock, False).startswith(b'HTTP/1.1 200'), 'first' + + time.sleep(3) + + assert req(sock, True).startswith(b'HTTP/1.1 200'), 'second' + + sock.close() + + finally: + min_rate_reset() + + +CONTINUE = b'HTTP/1.1 100 Continue\r\n\r\n' + + +def min_rate_expect(length): + # Sends a header with "Expect: 100-continue" and reads the 100. The + # client sends no body byte before it gets the 100. + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Expect: 100-continue\r\n' + + f'Content-Length: {length}\r\n'.encode() + + b'Connection: close\r\n\r\n' + ) + + data = b'' + sock.settimeout(5) + + while len(data) < len(CONTINUE): + try: + part = sock.recv(len(CONTINUE) - len(data)) + except socket.timeout: + break + + if not part: + break + + data += part + + assert data == CONTINUE, '100 before the body' + + return sock + + +def test_settings_body_min_rate_expect_slow_body(): + # After a 100 (Continue), the body read state starts in another + # function. The floor must be active there too. Otherwise a client + # can avoid the floor with "Expect: 100-continue". The client sends + # 1 byte each second after the 100 and must get 408. + client.load('empty') + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + try: + sock = min_rate_expect(10000) + sock.settimeout(1) + + start = time.monotonic() + data = b'' + closed = False + + while time.monotonic() - start < 12: + try: + sock.sendall(b'x') + except OSError: + closed = True + break + + try: + chunk = sock.recv(4096) + except socket.timeout: + continue + except OSError: + closed = True + break + + if not chunk: + closed = True + break + + data += chunk + + elapsed = time.monotonic() - start + sock.close() + + assert closed or data, 'slow body after 100 stopped' + assert data == b'' or data.startswith( + b'HTTP/1.1 408' + ), 'slow body after 100 408' + assert elapsed < 8, 'slow body after 100 stopped in time' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_expect_wait(): + # The client reads the 100, waits 1 s, and then sends the body much + # faster than the floor. The wait is shorter than body_read_timeout. + # Like the gap timer, the floor counts the wait in the first window. + # The client sends enough bytes before the first check, at 2 s, so + # it must get 200. + client.load('empty') + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + try: + sock = min_rate_expect(20000) + + time.sleep(1) + + try: + for _ in range(20): + sock.sendall(b'x' * 1000) + time.sleep(0.1) + + except OSError: + # The router answered early. The status shows why. + pass + + sock.settimeout(10) + + try: + data = sock.recv(4096) + except OSError: + data = b'' + + sock.close() + + assert data.startswith(b'HTTP/1.1 200'), 'wait after 100' + + finally: + min_rate_reset() + + +def test_settings_send_min_rate_slow_read(system, wait_for_record): + # A client reads a large response slower than send_min_rate, but fast + # enough that the router can write some data before each send_timeout. + # Each write starts the send_timeout gap timer again, so only the rate + # floor can stop this client. + # + # The router counts the bytes that the kernel accepts. On loopback the + # kernel send buffer grows to some MiB, and the kernel signals a write + # event only when about one third of the buffer is free. A client that + # reads only some bytes each second thus gets no write events, and the + # gap timer stops it. To test the floor, this test uses larger numbers: + # the floor is 8 MiB/s and the client reads about 1 MiB/s. The logic + # is the same as for a floor of 256 B/s on a real network. + # + # After the router closes the connection, the kernel still gives the + # client the data in the socket buffers. Thus the test finds the close + # in the router log. The time limits are generous because the kernel + # buffer sizes and the time of the write events are not exact. + # + # The numbers above are for the Linux loopback buffers. Other systems + # size and signal the send buffer in a different way. + if system != 'Linux': + pytest.skip('Linux loopback buffer sizes only') + + client.load('body_generate') + + min_rate_conf({'send_timeout': 3, 'send_min_rate': 8388608}) + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 262144) + sock.settimeout(1) + sock.connect(('127.0.0.1', 8080)) + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'X-Length: 67108864\r\n' + b'Connection: close\r\n\r\n' + ) + + start = time.monotonic() + total = 0 + closed = False + + while time.monotonic() - start < 12: + time.sleep(0.05) + + try: + chunk = sock.recv(65536) + except socket.timeout: + continue + except OSError: + closed = True + break + + if not chunk: + closed = True + break + + total += len(chunk) + + assert ( + wait_for_record( + r'client send rate is less than send_min_rate', wait=50 + ) + is not None + ), 'slow read stopped by the floor' + + # Read the data that the kernel keeps. Then the close must come. + sock.settimeout(5) + + try: + while not closed: + chunk = sock.recv(65536) + if not chunk: + closed = True + total += len(chunk) + except OSError: + closed = True + + sock.close() + + assert closed, 'slow read connection closed' + assert total < 67108864, 'slow read response cut' + + finally: + min_rate_reset() + + +def test_settings_send_min_rate_fast_read(): + # A client that reads at full speed must get all the data. Two + # responses on one keep-alive connection check that the second + # response starts with new rate state. + client.load('body_generate') + + min_rate_conf({'send_timeout': 2, 'send_min_rate': 256}) + + try: + for _ in range(2): + resp = client.get( + headers={ + 'Host': 'localhost', + 'X-Length': '4194304', + 'Connection': 'close', + }, + ) + assert resp['status'] == 200 + assert len(resp['body']) == 4194304 + + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(10) + + for close in (False, True): + time.sleep(3) + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'X-Length: 1048576\r\n' + + (b'Connection: close\r\n' if close else b'') + + b'\r\n' + ) + + data = b'' + while b'\r\n\r\n' not in data: + data += sock.recv(65536) + + head, body = data.split(b'\r\n\r\n', 1) + assert head.startswith(b'HTTP/1.1 200'), 'keep-alive status' + + while len(body) < 1048576: + chunk = sock.recv(65536) + assert chunk, 'keep-alive body' + body += chunk + + assert len(body) == 1048576, 'keep-alive length' + + sock.close() + + finally: + min_rate_reset() + + +def test_settings_send_min_rate_slow_application(): + # The send floor counts only the time when response data waits for the + # client. The time when the router waits for the application is not + # counted. Thus a slow application (for example a stream of events) + # is not stopped, also with a high floor. The application sends 4 + # parts with 1 s between them, which is longer than the grace time + # (send_timeout 2). The client reads at full speed. + client.load('delayed') + + min_rate_conf({'send_timeout': 2, 'send_min_rate': 8388608}) + + try: + body = '0123456789' * 400 + + resp = client.post( + headers={ + 'Host': 'localhost', + 'X-Parts': '4', + 'X-Delay': '1', + 'Connection': 'close', + }, + body=body, + read_timeout=30, + ) + + assert resp['status'] == 200, 'slow application status' + assert resp['body'] == body, 'slow application body' + + finally: + min_rate_reset() + + +def test_settings_send_min_rate_gap_longer_than_grace(search_in_file): + # The time between send periods is not counted, also when it is longer + # than the grace time. The application sends 1000 bytes, waits 2.5 s, + # and sends 1000 bytes more. The grace time is 2 s. The client reads + # at full speed, so the first period stops after about 1 ms. The second + # period starts again from its own write, and its first check sees about + # 2 ms, not 2.5 s. If the gap was counted, the check after the second + # write would see about 2000 bytes in 2.5 s, below the floor. That + # write sends the last data, so the client still gets the full body, + # but the router closes the connection. Thus the test sends a second + # request on the same connection. + client.load('send_bursts') + + min_rate_conf({'send_timeout': 2, 'send_min_rate': 1048576}) + + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.settimeout(10) + + def response(length): + data = b'' + while b'\r\n\r\n' not in data: + chunk = sock.recv(65536) + assert chunk, 'gap header' + data += chunk + + head, body = data.split(b'\r\n\r\n', 1) + assert head.startswith(b'HTTP/1.1 200'), 'gap status' + + while len(body) < length: + chunk = sock.recv(65536) + assert chunk, 'gap body' + body += chunk + + return body + + try: + start = time.monotonic() + + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'X-Part-Size: 1000\r\n' + b'X-Bursts: 2\r\n' + b'X-Interval: 2.5\r\n\r\n' + ) + + assert response(2000) == b'x' * 2000, 'gap body' + assert time.monotonic() - start >= 2.5, 'gap time' + + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'X-Part-Size: 10\r\n' + b'Connection: close\r\n\r\n' + ) + + assert response(10) == b'x' * 10, 'gap keep-alive body' + + assert ( + search_in_file(r'client send rate is less than send_min_rate') + is None + ), 'gap not stopped by the floor' + + finally: + sock.close() + min_rate_reset() + + +def test_settings_send_min_rate_stream_slow_read( + system, temp_dir, wait_for_record +): + # An application sends a stream in bursts, 1.5 s apart. The client + # keeps the socket send buffer full. Each 1.5 s it reads only one burst. + # Then the parts of the burst that wait can go out before send_timeout. + # Thus each write event empties the router write queue, and each send + # period stops there. The router must check the rate at that point + # too. Otherwise it never checks this client, and the client reads + # below the floor for as long as the stream lasts. + # + # The test uses a unix socket. On Linux, the kernel takes a unix socket + # write of 32 KiB whole or not at all, and the send buffer has the fixed + # size net.core.wmem_default. A write event comes when 3/4 of the + # buffer is free. Thus a part that does not fit waits whole, and the + # next write event sends all the parts that wait. No write sends only + # some of the queued data. + if system != 'Linux': + pytest.skip('Linux unix socket buffers only') + + # A new unix socket gets this size, the router socket too. A container + # can have no /proc/sys/net/core/wmem_default, so ask a socket. + pair = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + wmem = pair[0].getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) + pair[0].close() + pair[1].close() + + # The first burst fills the send buffer, and 2 or 3 parts wait. Each + # next burst is a little more than the space that a write event frees. + # The client reads one burst each time, so 2 or 3 parts wait in each + # cycle, for about 1 s. With wmem_default 212992: 9, then 6 parts. + part = 32768 + first = wmem // part + 3 + parts = wmem * 3 // 4 // part + 2 + bursts = 5 + length = part * (first + parts * (bursts - 1)) + + client.load('send_bursts') + + addr = f'{temp_dir}/sock' + + assert 'success' in client.conf( + {f'unix:{addr}': {'application': 'send_bursts'}}, 'listeners' + ) + + # The router measures about one burst for each 1 s of waiting. The + # floor is four bursts each second. + min_rate_conf({'send_timeout': 2, 'send_min_rate': 4 * parts * part}) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + start = time.monotonic() + + def read(size): + # Returns "size" bytes, or less if the connection closes. + data = b'' + + while len(data) < size: + try: + chunk = sock.recv(min(65536, size - len(data))) + except OSError: + break + + if not chunk: + break + + data += chunk + + return data + + try: + sock.settimeout(5) + sock.connect(addr) + sock.sendall( + b'GET / HTTP/1.1\r\n' + b'Host: localhost\r\n' + + f'X-First: {first}\r\n'.encode() + + f'X-Parts: {parts}\r\n'.encode() + + f'X-Part-Size: {part}\r\n'.encode() + + f'X-Bursts: {bursts}\r\n'.encode() + + b'X-Interval: 1.5\r\n' + b'Connection: close\r\n\r\n' + ) + start = time.monotonic() + + head = b'' + while not head.endswith(b'\r\n\r\n'): + byte = read(1) + if not byte: + break + + head += byte + + assert head.startswith(b'HTTP/1.1 200'), 'stream status' + + total = 0 + + for burst in range(bursts): + time.sleep(max(0, start + 1.5 * burst + 1 - time.monotonic())) + + size = parts * part if burst < bursts - 1 else length - total + got = len(read(size)) + total += got + + if got < size: + break + + assert total < length, 'slow stream reader stopped' + + assert ( + wait_for_record( + r'client send rate is less than send_min_rate', wait=50 + ) + is not None + ), 'slow stream reader stopped by the floor' + + finally: + sock.close() + min_rate_reset() + + # The application still runs its schedule. Let it end before the + # next test changes the configuration. + time.sleep(max(0, start + 7 - time.monotonic())) + + +def test_settings_min_rate_validation(): + client.load('empty') + + try: + for name in ('body_min_rate', 'send_min_rate'): + assert 'error' in client.conf({'http': {name: -1}}, 'settings') + assert 'error' in client.conf({'http': {name: 1.5}}, 'settings') + assert 'error' in client.conf({'http': {name: '256'}}, 'settings') + assert 'error' in client.conf( + {'http': {name: 2147483648}}, 'settings' + ) + assert 'success' in client.conf({'http': {name: 0}}, 'settings') + assert 'success' in client.conf( + {'http': {name: 2147483647}}, 'settings' + ) + assert 'success' in client.conf({'http': {name: 256}}, 'settings') + + finally: + min_rate_reset() + + def test_settings_idle_timeout(): client.load('empty') From 72b8d02416c35b3137345c400f0b3aa985283f9e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 3 Oct 2026 22:45:32 +0200 Subject: [PATCH 339/391] tests: wait 30 s for the echo of a large websocket message The 9_x tests wait 5 s for the echo after the client has sent the message. A debug build needs 3.6 to 4.4 s to echo 65536 fragments of 64 bytes. Under load the echo takes longer than 5 s and the test fails, although the echo arrives intact. Use one constant, ApplicationWebsocket.LARGE_MESSAGE_TIMEOUT, set to 30 s, in the asgi, java and node tests. The tests check the same things as before. Co-Authored-By: Claude Sonnet 5.5 --- test/test_asgi_websockets.py | 4 ++-- test/test_java_websockets.py | 2 +- test/test_node_websockets.py | 4 ++-- test/unit/applications/websockets.py | 4 ++++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index b9c24bc3a..11356fea6 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -1372,7 +1372,7 @@ def check_payload(opcode, length, chopsize=None): payload = b'*' * length ws.frame_write(sock, opcode, payload, chopsize=chopsize) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) def check_message(opcode, f_size): @@ -1382,7 +1382,7 @@ def check_message(opcode, f_size): payload = b'*' * 4 * 2**20 ws.message(sock, opcode, payload, fragmention_size=f_size) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) check_payload(op_text, 64 * 2**10) # 9_1_1 diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 97b966180..e56b709bb 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -1303,7 +1303,7 @@ def check_message(opcode, f_size): payload = b'*' * 4 * 2**20 ws.message(sock, opcode, payload, fragmention_size=f_size) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) check_payload(op_text, 64 * 2**10) # 9_1_1 diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index ccfa18748..1d4dd1184 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -1292,7 +1292,7 @@ def check_payload(opcode, length, chopsize=None): payload = b'*' * length ws.frame_write(sock, opcode, payload, chopsize=chopsize) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) def check_message(opcode, f_size): @@ -1302,7 +1302,7 @@ def check_message(opcode, f_size): payload = b'*' * 4 * 2**20 ws.message(sock, opcode, payload, fragmention_size=f_size) - frame = ws.frame_read(sock, read_timeout=5) + frame = ws.frame_read(sock, read_timeout=ws.LARGE_MESSAGE_TIMEOUT) check_frame(frame, True, opcode, payload) check_payload(op_text, 64 * 2**10) # 9_1_1 diff --git a/test/unit/applications/websockets.py b/test/unit/applications/websockets.py index a8e563d0f..0125c4e9e 100644 --- a/test/unit/applications/websockets.py +++ b/test/unit/applications/websockets.py @@ -14,6 +14,10 @@ class ApplicationWebsocket(ApplicationProto): + # Seconds to wait for the echo of a large message (the 9_x tests). + # A debug build needs 3.6-4.4 s to echo 65536 fragments of 64 bytes. + LARGE_MESSAGE_TIMEOUT = 30 + OP_CONT = 0x00 OP_TEXT = 0x01 OP_BINARY = 0x02 From 8c965dfc0087cf1efc6fa56748bb9b2e569c4d9c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 3 Oct 2026 20:13:40 +0200 Subject: [PATCH 340/391] http: answer 413 when a chunked body grows over max_body_size The router checks the size of a chunked request body twice. The check on the bytes that come with the header answers 413. The check on each later read called nxt_h1p_request_error(), which closes the connection with no status line. The client saw an empty reply or a reset. Now the later check sends 413 the same way, and clears keepalive because the rest of the body is never read. Two more failures on a later read of a chunked body closed the connection in the same way: an allocation failure in the chunk parser (h1p->chunked_parse.error) and a short write to the temporary file. Both now send 500 and clear keepalive, as their header-time equivalents in nxt_h1p_request_body_read() do. The 400 for a malformed chunk in a later read now clears keepalive and closes the connection, like the other error paths. A short write to the temporary file of a body with a Content-Length now answers 500 instead of closing silently, as its header-time equivalent does. Closes https://github.com/freeunitorg/freeunit/issues/542 Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 Reviewed-by: DeepSeek V4.1 Flash Harness: DeepSeek Harness --- CHANGES | 12 ++++++++ docs/changes.xml | 14 +++++++++ src/nxt_h1proto.c | 17 ++++++++--- test/test_chunked.py | 70 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index e172962d7..9e17d0583 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,18 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 + *) Bugfix: a chunked request body that grew over "max_body_size" + after the read that carried the request header got no response. + The router closed the connection without a status line. Now it + answers 413 and closes. A body that crossed the limit in the same + read as the header already got 413. A later read that fails to + allocate memory in the chunk parser, or to write the body to the + temporary file, now answers 500 and closes in the same way. The + 400 for a malformed chunk in a later read now clears keepalive + and closes the connection, like the other error paths. A short + write of a body with a Content-Length to the temporary file now + answers 500 instead of closing silently. + *) Bugfix: the router did not answer "Expect: 100-continue". A client that sent it, as curl and Guzzle do for large uploads, waited for its own timeout before it sent the body. For curl, that timeout is 1 diff --git a/docs/changes.xml b/docs/changes.xml index ef20b074c..1e927363a 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,20 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +a chunked request body that grew over "max_body_size" after the read that +carried the request header got no response. The router closed the connection +without a status line. Now it answers 413 and closes. A body that crossed the +limit in the same read as the header already got 413. A later read that fails +to allocate memory in the chunk parser, or to write the body to the temporary +file, now answers 500 and closes in the same way. The 400 for a malformed chunk +in a later read now clears keepalive and closes the connection, like the other +error paths. A short write of a body with a Content-Length to the temporary +file now answers 500 instead of closing silently. + + + the router did not answer "Expect: 100-continue". A client that sent it, as diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index febe9f0e0..0da22569b 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -1305,11 +1305,14 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) out = nxt_http_chunk_parse(task, &h1p->chunked_parse, b); if (h1p->chunked_parse.error) { - nxt_h1p_request_error(task, h1p, r); + h1p->keepalive = 0; + nxt_http_request_error(task, r, + NXT_HTTP_INTERNAL_SERVER_ERROR); return; } if (h1p->chunked_parse.chunk_error) { + h1p->keepalive = 0; nxt_http_request_error(task, r, NXT_HTTP_BAD_REQUEST); return; } @@ -1318,14 +1321,18 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) size = nxt_buf_mem_used_size(&chunk->mem); res = nxt_fd_write(b->file->fd, chunk->mem.pos, size); if (nxt_slow_path(res < (ssize_t) size)) { - nxt_h1p_request_error(task, h1p, r); + h1p->keepalive = 0; + nxt_http_request_error(task, r, + NXT_HTTP_INTERNAL_SERVER_ERROR); return; } b->file_end += size; if ((size_t) b->file_end > skcf->max_body_size) { - nxt_h1p_request_error(task, h1p, r); + h1p->keepalive = 0; + nxt_http_request_error(task, r, + NXT_HTTP_PAYLOAD_TOO_LARGE); return; } } @@ -1364,7 +1371,9 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) res = nxt_fd_write(b->file->fd, b->mem.pos, size); if (nxt_slow_path(res < (ssize_t) size)) { - nxt_h1p_request_error(task, h1p, r); + h1p->keepalive = 0; + nxt_http_request_error(task, r, + NXT_HTTP_INTERNAL_SERVER_ERROR); return; } diff --git a/test/test_chunked.py b/test/test_chunked.py index ea1743f70..eb9cb1e67 100644 --- a/test/test_chunked.py +++ b/test/test_chunked.py @@ -97,6 +97,76 @@ def test_chunked_max_body_size(): ) +def test_chunked_max_body_size_later_read(): + # The body crosses the limit in a read after the header read. That + # check closed the connection with no status line. + # https://github.com/freeunitorg/freeunit/issues/542 + assert 'success' in client.conf( + {'max_body_size': 8, 'chunked_transform': True}, 'settings/http' + ) + + def send(writes, connection_close): + head = ( + b'POST / HTTP/1.1\r\nHost: localhost\r\n' + b'Transfer-Encoding: chunked\r\n' + ) + + if connection_close: + head += b'Connection: close\r\n' + + sock = client.http(head + b'\r\n', raw=True, no_recv=True) + + # Each write ends with the CRLF after the chunk data, and goes in + # its own read. + for w in writes: + time.sleep(0.3) + sock.sendall(w) + + resp = client.recvall(sock, read_timeout=10) + sock.close() + + return resp + + # The request does not ask to close, so "Connection: close" in the + # response shows that keep-alive was turned off. The rest of the body + # is never read. + resp = send([b'5\r\nhello\r\n', b'6\r\nworld!\r\n'], False) + assert resp.startswith(b'HTTP/1.1 413 '), resp + assert b'\r\nConnection: close\r\n' in resp, resp + + # Control: exactly 8 bytes is not over the limit. + resp = send([b'5\r\nhello\r\n', b'3\r\nabc\r\n', b'0\r\n\r\n'], True) + assert resp.startswith(b'HTTP/1.1 200 '), resp + assert resp.endswith(b'\r\n\r\nhelloabc'), resp + + +def test_chunked_malformed_later_read(): + # A malformed chunk in a read after the header read got 400, but the + # connection stayed open for a next request. + sock = client.http( + b'POST / HTTP/1.1\r\nHost: localhost\r\n' + b'Transfer-Encoding: chunked\r\n\r\n', + raw=True, + no_recv=True, + ) + + # "X" stands where the CRLF after the chunk data must be. + time.sleep(0.3) + sock.sendall(b'5\r\nhelloXXXXXXXX') + + start = time.monotonic() + resp = client.recvall(sock, read_timeout=5) + elapsed = time.monotonic() - start + sock.close() + + assert resp.startswith(b'HTTP/1.1 400 '), resp + assert b'\r\nConnection: close\r\n' in resp, resp + + # The server closes the connection. recvall() returns at the end of + # the stream, not at its read timeout. + assert elapsed < 3, elapsed + + def test_chunked_after_last(): resp = client.get( headers={ From d6a67074b637d6d447169a8269451fd4c94ec95a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 00:20:01 +0200 Subject: [PATCH 341/391] regex: limit the recursion depth on PCRE2 before 10.30 PCRE2 before 10.30 makes the nested calls of a match on the stack, as PCRE 1 does. https://github.com/freeunitorg/freeunit/pull/520 gave the PCRE 1 path a recursion limit, but not these versions. Their library default is 10,000,000 calls. Amazon Linux 2 ships PCRE2 10.23. In PCRE2 10.23 of Amazon Linux 2 on x86-64, one nested call uses 448 bytes of stack, and a repeated group nests 2 calls per byte. So a long subject can use up a small router thread stack. The match context now sets a recursion limit of 2,000 calls on PCRE2 before 10.30. That is about 900 KB of stack, and the same limit as on PCRE 1. A match that reaches it fails, the error log gets a warning with the pattern and the subject length, and the request gets a 500 response. So on these versions, a repeated group on a subject of more than about 1,000 bytes gets a 500 response, as with PCRE 1. PCRE2 10.30 and later keep the backtracking frames on the heap, and nothing changes for them except the log. There, the same error code is the depth limit. Unit does not set it, and its library default is 10,000,000. If a library is built with a lower default, a match that reaches it now logs the warning, not 64 bytes of the subject. The unit test now expects the 8 KiB repeated group to fail on PCRE2 before 10.30, as on PCRE 1. Without the limit, the match returned 1. CHANGES and docs/changes.xml said that a match stops "after 100,000 steps". A pattern that is not anchored stops at 100,000 callouts, which is sooner. The text now says "after at most 100,000 steps". It also names PCRE2 before 10.30 next to PCRE 1. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- CHANGES | 21 +++++++++++---------- docs/changes.xml | 20 ++++++++++---------- src/nxt_pcre2.c | 17 +++++++++++++++++ src/test/nxt_regex_test.c | 17 +++++++++++------ 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/CHANGES b/CHANGES index 9e17d0583..f43767e1f 100644 --- a/CHANGES +++ b/CHANGES @@ -56,16 +56,17 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 *) Bugfix: a regular expression in the configuration could run for up to 10,000,000 match steps on one request. A match now stops after - 100,000 steps and the request gets a 500 response. The error log - then gets a warning with the pattern and the subject length, not - the subject. In "compression" "types", a match that stops is not a - match, and the response is not compressed. A repeated group takes - about 2 steps per byte. So a subject much longer than 8 KiB can - reach the limit and get a 500 response. Such a subject needs a - "large_header_buffer_size" above its default of 8192. With PCRE 1, - a match also stops at 2,000 nested calls and does not overflow the - thread stack. There, a repeated group on a subject of more than - about 1,000 bytes can get a 500 response. + at most 100,000 steps and the request gets a 500 response. The + error log then gets a warning with the pattern and the subject + length, not the subject. In "compression" "types", a match that + stops is not a match, and the response is not compressed. A + repeated group takes about 2 steps per byte. So a subject much + longer than 8 KiB can reach the limit and get a 500 response. Such + a subject needs a "large_header_buffer_size" above its default of + 8192. With PCRE 1 and with PCRE2 before 10.30, a match also stops + at 2,000 nested calls and does not overflow the thread stack. + There, a repeated group on a subject of more than about 1,000 + bytes can get a 500 response. *) Security: the router and libunit read records from shared memory that the other process can still write, without enough checks. A diff --git a/docs/changes.xml b/docs/changes.xml index 1e927363a..8ead6e729 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -83,16 +83,16 @@ store to finish. When the store fails, the main process logs an alert. a regular expression in the configuration could run for up to 10,000,000 -match steps on one request. A match now stops after 100,000 steps and the -request gets a 500 response. The error log then gets a warning with the -pattern and the subject length, not the subject. In "compression" "types", a -match that stops is not a match, and the response is not compressed. A -repeated group takes about 2 steps per byte. So a subject much longer than -8 KiB can reach the limit and get a 500 response. Such a subject needs a -"large_header_buffer_size" above its default of 8192. With PCRE 1, a match -also stops at 2,000 nested calls and does not overflow the thread stack. -There, a repeated group on a subject of more than about 1,000 bytes can get a -500 response. +match steps on one request. A match now stops after at most 100,000 steps +and the request gets a 500 response. The error log then gets a warning with +the pattern and the subject length, not the subject. In "compression" +"types", a match that stops is not a match, and the response is not +compressed. A repeated group takes about 2 steps per byte. So a subject +much longer than 8 KiB can reach the limit and get a 500 response. Such a +subject needs a "large_header_buffer_size" above its default of 8192. With +PCRE 1 and with PCRE2 before 10.30, a match also stops at 2,000 nested calls +and does not overflow the thread stack. There, a repeated group on a subject +of more than about 1,000 bytes can get a 500 response. diff --git a/src/nxt_pcre2.c b/src/nxt_pcre2.c index 35e353dec..387f6fbc6 100644 --- a/src/nxt_pcre2.c +++ b/src/nxt_pcre2.c @@ -21,6 +21,15 @@ */ #define NXT_PCRE2_HEAP_LIMIT 16384 +/* + * The most nested calls that one match may make with PCRE2 before 10.30. + * These versions make the calls on the stack, 448 bytes each in PCRE2 10.23 + * of Amazon Linux 2 on x86-64. 2,000 calls use about 900 KB. A request + * matches on a router thread with the default glibc stack: 8 MB, or 2 MB + * when the stack rlimit is unlimited. The library default is 10,000,000. + */ +#define NXT_PCRE2_RECURSION_LIMIT 2000 + static void *nxt_pcre2_malloc(PCRE2_SIZE size, void *memory_data); static void nxt_pcre2_free(void *p, void *memory_data); @@ -245,6 +254,8 @@ nxt_regex_match_create(nxt_mp_t *mp, size_t size) #if (PCRE2_MAJOR > 10 || PCRE2_MINOR >= 30) (void) pcre2_set_heap_limit(ctx, NXT_PCRE2_HEAP_LIMIT); +#else + (void) pcre2_set_recursion_limit(ctx, NXT_PCRE2_RECURSION_LIMIT); #endif (void) pcre2_set_callout(ctx, nxt_pcre2_callout, match); @@ -290,6 +301,12 @@ nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, break; #endif + /* PCRE2 10.30 and later call it the depth limit. */ + + case PCRE2_ERROR_RECURSIONLIMIT: + limit = "recursion"; + break; + default: limit = NULL; break; diff --git a/src/test/nxt_regex_test.c b/src/test/nxt_regex_test.c index 868898878..0e2093de3 100644 --- a/src/test/nxt_regex_test.c +++ b/src/test/nxt_regex_test.c @@ -7,11 +7,11 @@ * takes about 2M match steps on 2000 separators on PCRE2 10.4x and PCRE * 8.39. The match must stop at NXT_REGEX_MATCH_LIMIT and report an error. * - * A repeated group takes about 2 match steps per byte. On PCRE2 it must - * still match on an 8 KiB subject. PCRE 1 also nests about 2 calls per - * byte on the stack. On an 8 KiB subject it must stop at its recursion - * limit and report an error, not overflow the stack. On a 512-byte subject - * it must match. + * A repeated group takes about 2 match steps per byte. On PCRE2 10.30 and + * later it must still match on an 8 KiB subject. PCRE 1 and PCRE2 before + * 10.30 also nest about 2 calls per byte on the stack. On an 8 KiB subject + * they must stop at the recursion limit and report an error, not overflow + * the stack. On a 512-byte subject they must match. * * The library counts match steps from zero again at each start position of * a pattern that is not anchored. In PCRE2 10.42, "(?=a)(?:a|aa){0,12}[xy]" @@ -26,12 +26,17 @@ #include #include "nxt_tests.h" +#if (NXT_HAVE_PCRE2) +#define PCRE2_CODE_UNIT_WIDTH 8 +#include +#endif + #define NXT_REGEX_TEST_SIZE 8192 #define NXT_REGEX_TEST_SHORT 512 #define NXT_REGEX_TEST_STARTS 1000 -#if (NXT_HAVE_PCRE2) +#if (NXT_HAVE_PCRE2 && (PCRE2_MAJOR > 10 || PCRE2_MINOR >= 30)) #define NXT_REGEX_TEST_LONG_RESULT 1 #else #define NXT_REGEX_TEST_LONG_RESULT NXT_ERROR From 1c66c4d470bd620b692baedcbd3780875b30a4bf Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 06:26:27 +0200 Subject: [PATCH 342/391] port: never hand out RPC stream 0 after the counter wraps Stream 0 means "no handler" to every caller of nxt_port_rpc_register_handler() and to the router main port. After 2^32 registrations the shared counter returned 0. The caller saw a failure, and the registration stayed. nxt_port_rpc_register_handler_ex() now takes the next value again when it gets 0. The test sets the counter to UINT32_MAX through a new NXT_TESTS hook and registers twice. It restores the old counter value afterwards. Without the fix the test logs "rpc stream wrap: 4294967295 then 0". Co-Authored-By: Claude Opus 5.5 --- CHANGES | 4 +++ docs/changes.xml | 8 ++++++ src/nxt_port.c | 18 ++++++------- src/nxt_port_rpc.c | 33 ++++++++++++++++++++--- src/nxt_port_rpc.h | 1 + src/test/nxt_port_fail_test.c | 49 +++++++++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 13 deletions(-) diff --git a/CHANGES b/CHANGES index f43767e1f..187cde8e8 100644 --- a/CHANGES +++ b/CHANGES @@ -78,6 +78,10 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 marker that the application can clear. All of these are now checked. + *) Bugfix: after 2^32 RPC registrations the stream counter wrapped + and handed out stream 0, which callers read as a failure. The + registration stayed. Stream 0 is now skipped. + *) Bugfix: a keep-alive TLS connection that was idle across a reconfiguration could use the TLS settings of the old configuration after they were freed, when it was closed. The connection now keeps diff --git a/docs/changes.xml b/docs/changes.xml index 8ead6e729..f42d18f2e 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,14 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +after 2^32 RPC registrations the stream counter wrapped and handed out +stream 0, which callers read as a failure. The registration stayed. +Stream 0 is now skipped. + + + a chunked request body that grew over "max_body_size" after the read that diff --git a/src/nxt_port.c b/src/nxt_port.c index b88a6fba9..29ad3e602 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -1013,16 +1013,16 @@ nxt_port_process_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) * set, nxt_port_remove_notify_others() puts it into the REMOVE_PID that * reports this process's death, and nxt_router_remove_pid_handler() * turns a stream-bearing REMOVE_PID into an RPC_ERROR - * (src/nxt_router.c:1147-1153). That is the right fallback for a start + * (src/nxt_router.c). That is the right fallback for a start * that never completed, and a liability afterwards: stream identifiers - * come from one 32-bit counter (nxt_stream_ident, src/nxt_port_rpc.c:11, - * bumped at src/nxt_port_rpc.c:164) that every request also draws on - * (src/nxt_router.c:5880), so once it wraps, an ordinary worker exit - * would fail whatever live RPC has inherited the number. The reachable - * collision set is small -- the retype lands on the router's main port, - * which holds start, prefork, listen-socket and access-log - * registrations, while request RPCs live on the worker threads' engine - * ports -- but it is not empty. + * come from one 32-bit counter (nxt_stream_ident, src/nxt_port_rpc.c, + * bumped in nxt_port_rpc_register_handler_ex()) that every request also + * draws on (nxt_router_process_http_request()), so once it wraps, an + * ordinary worker exit would fail whatever live RPC has inherited the + * number. The reachable collision set is small -- the retype lands on + * the router's main port, which holds start, prefork, listen-socket and + * access-log registrations, while request RPCs live on the worker + * threads' engine ports -- but it is not empty. * * Zeroing on the READY state alone is what this deliberately is not. * The state is set above and the announcement is sent here, and in diff --git a/src/nxt_port_rpc.c b/src/nxt_port_rpc.c index 1ddf9198c..9a5f295ac 100644 --- a/src/nxt_port_rpc.c +++ b/src/nxt_port_rpc.c @@ -51,6 +51,24 @@ nxt_port_rpc_test_insert_failures(nxt_uint_t failures) } +/* + * Set the next stream identifier and return the previous one. A test uses + * it to reach the 32-bit wrap and to restore the counter afterwards. It is + * a plain store: the tests are single-threaded. + */ + +uint32_t +nxt_port_rpc_test_set_stream_ident(uint32_t stream) +{ + uint32_t old; + + old = *nxt_stream_ident; + *nxt_stream_ident = stream; + + return old; +} + + static nxt_bool_t nxt_port_rpc_test_should_fail(nxt_uint_t *failures) { @@ -158,9 +176,16 @@ nxt_port_rpc_register_handler_ex(nxt_task_t *task, nxt_port_t *port, nxt_port_rpc_handler_t ready_handler, nxt_port_rpc_handler_t error_handler, size_t ex_size) { + uint32_t stream; + + /* Stream 0 means "no handler" to the callers: skip it after a wrap. */ + + do { + stream = nxt_atomic_fetch_add(nxt_stream_ident, 1); + } while (nxt_slow_path(stream == 0)); + return nxt_port_rpc_reg_add(task, port, ready_handler, error_handler, - nxt_atomic_fetch_add(nxt_stream_ident, 1), - ex_size); + stream, ex_size); } @@ -178,8 +203,8 @@ nxt_port_rpc_register_handler_ex(nxt_task_t *task, nxt_port_t *port, * and does not delete again afterwards -- so the key is free at that point, * and nothing removes what this puts back. Anywhere else the insert would * either collide with a live registration or be undone by the caller's own - * cleanup, and the stream identifier itself is only unique because - * nxt_stream_ident is never reused. + * cleanup, and the stream identifier itself is only unique until + * nxt_stream_ident wraps. */ void * diff --git a/src/nxt_port_rpc.h b/src/nxt_port_rpc.h index b30140b4e..1f2de1dc0 100644 --- a/src/nxt_port_rpc.h +++ b/src/nxt_port_rpc.h @@ -26,6 +26,7 @@ void *nxt_port_rpc_register_handler_at(nxt_task_t *task, nxt_port_t *port, #if (NXT_TESTS) void nxt_port_rpc_test_alloc_failures(nxt_uint_t failures); void nxt_port_rpc_test_insert_failures(nxt_uint_t failures); +uint32_t nxt_port_rpc_test_set_stream_ident(uint32_t stream); #endif uint32_t nxt_port_rpc_ex_stream(void *ex); diff --git a/src/test/nxt_port_fail_test.c b/src/test/nxt_port_fail_test.c index c13c34ca5..d91e29fea 100644 --- a/src/test/nxt_port_fail_test.c +++ b/src/test/nxt_port_fail_test.c @@ -1128,6 +1128,51 @@ nxt_port_fail_test_quit_log_level(nxt_thread_t *thr) } +/* + * The counter wraps: the stream after UINT32_MAX must not be 0. The port + * is open, and the registrations are cancelled before the return. + */ + +static nxt_int_t +nxt_port_fail_test_rpc_stream_wrap(nxt_task_t *task, nxt_port_t *port) +{ + void *ex; + uint32_t stream[2], saved_stream; + + stream[0] = 0; + stream[1] = 0; + + saved_stream = nxt_port_rpc_test_set_stream_ident(UINT32_MAX); + + ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); + if (ex != NULL) { + stream[0] = nxt_port_rpc_ex_stream(ex); + + ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); + if (ex != NULL) { + stream[1] = nxt_port_rpc_ex_stream(ex); + nxt_port_rpc_cancel(task, port, stream[1]); + } + + nxt_port_rpc_cancel(task, port, stream[0]); + } + + nxt_port_rpc_test_set_stream_ident(saved_stream); + + if (stream[0] != UINT32_MAX || stream[1] == 0 + || port->use_count != 1 || !nxt_lvlhsh_is_empty(&port->rpc_streams)) + { + nxt_log_error(NXT_LOG_NOTICE, task->log, + "port failure test rpc stream wrap: %uD then %uD, " + "use_count %D", stream[0], stream[1], + (int32_t) port->use_count); + return NXT_ERROR; + } + + return NXT_OK; +} + + static nxt_int_t nxt_port_fail_test_rpc_register(nxt_thread_t *thr) { @@ -1149,6 +1194,10 @@ nxt_port_fail_test_rpc_register(nxt_thread_t *thr) port->pair[0] = 0; + if (nxt_port_fail_test_rpc_stream_wrap(task, port) != NXT_OK) { + goto fail; + } + nxt_port_rpc_test_alloc_failures(1); ex = nxt_port_rpc_register_handler_ex(task, port, NULL, NULL, 0); From 1dd9af5ca7e7c45283002785568c579b63f28e2a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 06:38:41 +0200 Subject: [PATCH 343/391] wasm: check the guest malloc result against the linear memory size The offset from the guest's malloc handler was stored as int32 into a size_t. A negative value put the host base pointer below the linear memory. No check compared the offset with the memory size. The host now reads the offset as uint32 and keeps it in a uint32 field. It refuses an offset that does not leave the number of bytes it asked the guest for. It also refuses an address that is not aligned for nxt_wasm_request_t, which has uint64_t members. In both cases the worker exits at startup. The test guest has five malloc handlers. Four return an offset that the host refuses: a negative offset, an offset past the memory end, an odd offset, and an offset 8 bytes past the last valid one. The odd offset and the offset past the last valid one come after a memory grow, so only the alignment check refuses the first and only the offset check refuses the second. The router answers 503 for all four. The fifth handler returns the last valid offset, and the request gets 200. Co-Authored-By: Claude Opus 5.5 Reviewed-by: DeepSeek V4.1 Flash --- CHANGES | 6 +++ docs/changes.xml | 9 ++++ src/wasm/nxt_rt_wasmtime.c | 39 ++++++++++++-- src/wasm/nxt_wasm.h | 2 +- test/test_wasm.py | 43 ++++++++++++++++ test/unit/applications/lang/wasm.py | 10 +++- test/wasm/badmalloc/badmalloc.c | 80 +++++++++++++++++++++++++++++ 7 files changed, 182 insertions(+), 7 deletions(-) create mode 100644 test/wasm/badmalloc/badmalloc.c diff --git a/CHANGES b/CHANGES index 187cde8e8..82e05ba8c 100644 --- a/CHANGES +++ b/CHANGES @@ -68,6 +68,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 There, a repeated group on a subject of more than about 1,000 bytes can get a 500 response. + *) Security: the wasm module did not check the offset that the malloc + handler of the guest returned. A negative or large offset put the + request buffer outside the linear memory of the guest. The worker + now refuses such an offset, or an offset that is not aligned, and + exits at startup. + *) Security: the router and libunit read records from shared memory that the other process can still write, without enough checks. A buffer that was not a whole number of records was read past its diff --git a/docs/changes.xml b/docs/changes.xml index f42d18f2e..f60351347 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -88,6 +88,15 @@ store to finish. When the store fails, the main process logs an alert. + + +the wasm module did not check the offset that the malloc handler of the guest +returned. A negative or large offset put the request buffer outside the +linear memory of the guest. The worker now refuses such an offset, or an +offset that is not aligned, and exits at startup. + + + a regular expression in the configuration could run for up to 10,000,000 diff --git a/src/wasm/nxt_rt_wasmtime.c b/src/wasm/nxt_rt_wasmtime.c index 64ef775de..a1945d2cc 100644 --- a/src/wasm/nxt_rt_wasmtime.c +++ b/src/wasm/nxt_rt_wasmtime.c @@ -305,6 +305,9 @@ nxt_wasmtime_init_memory(nxt_wasm_ctx_t *ctx) { int i = 0; bool ok; + size_t need, size; + uint8_t *base; + uint32_t off; wasm_trap_t *trap = NULL; wasmtime_val_t args[1] = { }; wasmtime_val_t results[1] = { }; @@ -313,8 +316,10 @@ nxt_wasmtime_init_memory(nxt_wasm_ctx_t *ctx) nxt_wasmtime_ctx_t *rt_ctx = &nxt_wasmtime_ctx; const nxt_wasm_func_t *func = &ctx->fh[NXT_WASM_FH_MALLOC].func; + need = NXT_WASM_MEM_SIZE + NXT_WASM_PAGE_SIZE; + args[i].kind = WASMTIME_I32; - args[i++].of.i32 = NXT_WASM_MEM_SIZE + NXT_WASM_PAGE_SIZE; + args[i++].of.i32 = need; error = wasmtime_func_call(rt_ctx->ctx, func, args, i, results, 1, &trap); if (error != NULL || trap != NULL) { @@ -332,10 +337,36 @@ nxt_wasmtime_init_memory(nxt_wasm_ctx_t *ctx) } rt_ctx->memory = item.of.memory; - ctx->baddr_off = results[0].of.i32; - ctx->baddr = wasmtime_memory_data(rt_ctx->ctx, &rt_ctx->memory); + /* + * The guest returns a wasm32 pointer. Read it as unsigned, and check + * that the bytes the host asked for fit in the linear memory from it. + */ + off = (uint32_t) results[0].of.i32; + size = wasmtime_memory_data_size(rt_ctx->ctx, &rt_ctx->memory); + + if (size < need || off > size - need) { + nxt_wasmtime_err_msg(NULL, NULL, + "malloc handler returned offset %u outside " + "memory of %zu bytes", off, size); + return -1; + } + + base = wasmtime_memory_data(rt_ctx->ctx, &rt_ctx->memory); + + /* + * The host writes an nxt_wasm_request_t at this address. The structure + * has uint64_t members, so the address must be aligned for it. + */ + if ((uintptr_t) (base + off) % _Alignof(nxt_wasm_request_t) != 0) { + nxt_wasmtime_err_msg(NULL, NULL, + "malloc handler returned offset %u that is not " + "aligned to %zu bytes", off, + _Alignof(nxt_wasm_request_t)); + return -1; + } - ctx->baddr += ctx->baddr_off; + ctx->baddr_off = off; + ctx->baddr = base + off; return 0; } diff --git a/src/wasm/nxt_wasm.h b/src/wasm/nxt_wasm.h index 6bc3ae35a..25b79605c 100644 --- a/src/wasm/nxt_wasm.h +++ b/src/wasm/nxt_wasm.h @@ -117,7 +117,7 @@ struct nxt_wasm_ctx_s { nxt_unit_request_info_t *req; uint8_t *baddr; - size_t baddr_off; + uint32_t baddr_off; size_t response_off; diff --git a/test/test_wasm.py b/test/test_wasm.py index f7f7dcd86..ca7935290 100644 --- a/test/test_wasm.py +++ b/test/test_wasm.py @@ -1,5 +1,7 @@ from pathlib import Path +import pytest + from unit.applications.lang.wasm import ApplicationWasm from unit.check.check_prerequisites import check_prerequisites from unit.option import option @@ -61,3 +63,44 @@ def test_wasm_access_filesystem_denied(): resp = client.get(url=url) assert resp['status'] == 404, url + + +def test_wasm_malloc_handler_real_malloc(): + client.load('hello', malloc_handler='malloc') + + resp = client.get() + + assert resp['status'] == 200 + assert resp['body'] == 'Hello from wasm\n' + + +def test_wasm_malloc_at_memory_end(): + client.load('badmalloc', malloc_handler='edge_malloc') + + resp = client.get() + + assert resp['status'] == 200 + assert resp['body'] == 'accepted\n' + + +@pytest.mark.parametrize( + 'handler', ['bad_malloc_neg', 'bad_malloc_big', 'bad_malloc_end'] +) +def test_wasm_malloc_out_of_range(handler, wait_for_record): + client.load('badmalloc', malloc_handler=handler) + + assert client.get()['status'] == 503 + + assert ( + wait_for_record(r'malloc handler returned offset \d+ outside memory') + is not None + ) + + +def test_wasm_malloc_unaligned(wait_for_record): + client.load('badmalloc', malloc_handler='bad_malloc_odd') + + assert client.get()['status'] == 503 + + record = r'malloc handler returned offset \d+ that is not aligned' + assert wait_for_record(record) is not None diff --git a/test/unit/applications/lang/wasm.py b/test/unit/applications/lang/wasm.py index 9d627a7fa..b05d0264c 100644 --- a/test/unit/applications/lang/wasm.py +++ b/test/unit/applications/lang/wasm.py @@ -71,7 +71,13 @@ def prepare_env(script): return output - def load(self, script, access_filesystem=None, **kwargs): + def load( + self, + script, + access_filesystem=None, + malloc_handler='luw_malloc_handler', + **kwargs, + ): module = self.prepare_env(script) app = { @@ -79,7 +85,7 @@ def load(self, script, access_filesystem=None, **kwargs): 'processes': {'spare': 0}, 'module': str(module), 'request_handler': 'luw_request_handler', - 'malloc_handler': 'luw_malloc_handler', + 'malloc_handler': malloc_handler, 'free_handler': 'luw_free_handler', # Without these two the guest's init hook never runs, so # request_buf stays NULL and luw_set_req_buf() memcpy()s to diff --git a/test/wasm/badmalloc/badmalloc.c b/test/wasm/badmalloc/badmalloc.c new file mode 100644 index 000000000..56a721dd3 --- /dev/null +++ b/test/wasm/badmalloc/badmalloc.c @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright (C) FreeUnit contributors. */ + +/* + * A guest whose malloc handlers return offsets that the host must check. + * The host refuses a bad offset when the worker starts, so the request + * handler runs only for "edge_malloc". + */ + +#include "unit/unit-wasm.h" + +#define WASM_PAGE_SIZE 65536u +#define RESPONSE_OFFSET 4096 + +/* + * Adds at least "bytes" of new pages to the linear memory and returns its + * new size. The handlers below return offsets in the new pages, so the + * host never writes over the data or the stack of the guest. + */ +static u32 +grow_by(u32 bytes) +{ + __builtin_wasm_memory_grow(0, (bytes + WASM_PAGE_SIZE - 1) + / WASM_PAGE_SIZE); + + return (u32) __builtin_wasm_memory_size(0) * WASM_PAGE_SIZE; +} + +__luw_export_name("bad_malloc_neg") +u32 bad_malloc_neg(size_t size) +{ + (void) size; + + return 0x80000000u; +} + +__luw_export_name("bad_malloc_big") +u32 bad_malloc_big(size_t size) +{ + (void) size; + + return 0x7fff0000u; +} + +/* The last offset that leaves "size" bytes. The host accepts it. */ +__luw_export_name("edge_malloc") +u32 edge_malloc(size_t size) +{ + return grow_by(size) - size; +} + +/* + * The memory is larger than "size", but the offset leaves 8 bytes too few. + * The offset is aligned, so only the offset check refuses it. + */ +__luw_export_name("bad_malloc_end") +u32 bad_malloc_end(size_t size) +{ + return grow_by(size) - size + 8; +} + +/* The offset leaves enough bytes, but it is odd. */ +__luw_export_name("bad_malloc_odd") +u32 bad_malloc_odd(size_t size) +{ + return grow_by(size + WASM_PAGE_SIZE) - size - WASM_PAGE_SIZE + 1; +} + +__luw_export_name("luw_request_handler") +int luw_request_handler(u8 *addr) +{ + luw_ctx_t ctx; + + luw_init_ctx(&ctx, addr, RESPONSE_OFFSET); + luw_mem_writep(&ctx, "accepted\n"); + luw_http_send_response(&ctx); + luw_http_response_end(); + + return 0; +} From 0d96be58abdcc4c1ead65ad726ab96e0eac6b87c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 11:21:33 +0200 Subject: [PATCH 344/391] docker: build the images with Rust 1.95.0 Wasmtime 48 needs Rust 1.95.0. The images pinned 1.94.1, so the wasm image could not build the wasm-wasi-component module or the Wasmtime C API after the Wasmtime update. The other images use Rust only for the otel crate, which needs 1.88. pkg/docker/template.Dockerfile sets the new version. The 27 Dockerfiles come from "make dockerfiles"; each changes only in RUST_VERSION. The local builder images (freeunit-builder:trixie-rust1.94.1 and php8.5-rust1.94.1) still carry 1.94.1. They are published separately. Co-Authored-By: Claude Opus 5.5 --- pkg/docker/Dockerfile.go-1.25 | 2 +- pkg/docker/Dockerfile.go-1.26 | 2 +- pkg/docker/Dockerfile.java-17 | 2 +- pkg/docker/Dockerfile.java-21 | 2 +- pkg/docker/Dockerfile.java-25 | 2 +- pkg/docker/Dockerfile.java-26 | 2 +- pkg/docker/Dockerfile.minimal | 2 +- pkg/docker/Dockerfile.node-20 | 2 +- pkg/docker/Dockerfile.node-22 | 2 +- pkg/docker/Dockerfile.node-24 | 2 +- pkg/docker/Dockerfile.node-26 | 2 +- pkg/docker/Dockerfile.perl-5.38 | 2 +- pkg/docker/Dockerfile.perl-5.40 | 2 +- pkg/docker/Dockerfile.perl-5.42 | 2 +- pkg/docker/Dockerfile.php-8.3 | 2 +- pkg/docker/Dockerfile.php-8.4 | 2 +- pkg/docker/Dockerfile.php-8.5 | 2 +- pkg/docker/Dockerfile.python-3.12 | 2 +- pkg/docker/Dockerfile.python-3.12-slim | 2 +- pkg/docker/Dockerfile.python-3.13 | 2 +- pkg/docker/Dockerfile.python-3.13-slim | 2 +- pkg/docker/Dockerfile.python-3.14 | 2 +- pkg/docker/Dockerfile.python-3.14-slim | 2 +- pkg/docker/Dockerfile.ruby-3.3 | 2 +- pkg/docker/Dockerfile.ruby-3.4 | 2 +- pkg/docker/Dockerfile.ruby-4.0 | 2 +- pkg/docker/Dockerfile.wasm | 2 +- pkg/docker/template.Dockerfile | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/pkg/docker/Dockerfile.go-1.25 b/pkg/docker/Dockerfile.go-1.25 index 535c7f684..93ed5cac6 100644 --- a/pkg/docker/Dockerfile.go-1.25 +++ b/pkg/docker/Dockerfile.go-1.25 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.go-1.26 b/pkg/docker/Dockerfile.go-1.26 index a26d228eb..e13cad5a1 100644 --- a/pkg/docker/Dockerfile.go-1.26 +++ b/pkg/docker/Dockerfile.go-1.26 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.java-17 b/pkg/docker/Dockerfile.java-17 index 12e935912..bd1919ae7 100644 --- a/pkg/docker/Dockerfile.java-17 +++ b/pkg/docker/Dockerfile.java-17 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.java-21 b/pkg/docker/Dockerfile.java-21 index 9d1da9ea8..212cc152a 100644 --- a/pkg/docker/Dockerfile.java-21 +++ b/pkg/docker/Dockerfile.java-21 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.java-25 b/pkg/docker/Dockerfile.java-25 index 555a37af2..79393f4d5 100644 --- a/pkg/docker/Dockerfile.java-25 +++ b/pkg/docker/Dockerfile.java-25 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.java-26 b/pkg/docker/Dockerfile.java-26 index 662519e60..717f10d7c 100644 --- a/pkg/docker/Dockerfile.java-26 +++ b/pkg/docker/Dockerfile.java-26 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index 2b2671742..f8e5ae895 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.node-20 b/pkg/docker/Dockerfile.node-20 index e2b6d7272..fccba2982 100644 --- a/pkg/docker/Dockerfile.node-20 +++ b/pkg/docker/Dockerfile.node-20 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.node-22 b/pkg/docker/Dockerfile.node-22 index 69849a1d8..8f0c640e0 100644 --- a/pkg/docker/Dockerfile.node-22 +++ b/pkg/docker/Dockerfile.node-22 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.node-24 b/pkg/docker/Dockerfile.node-24 index f0fc9737a..e5ab23cb7 100644 --- a/pkg/docker/Dockerfile.node-24 +++ b/pkg/docker/Dockerfile.node-24 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.node-26 b/pkg/docker/Dockerfile.node-26 index fae55c414..0614cb299 100644 --- a/pkg/docker/Dockerfile.node-26 +++ b/pkg/docker/Dockerfile.node-26 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.perl-5.38 b/pkg/docker/Dockerfile.perl-5.38 index 79023c465..482a4820c 100644 --- a/pkg/docker/Dockerfile.perl-5.38 +++ b/pkg/docker/Dockerfile.perl-5.38 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.perl-5.40 b/pkg/docker/Dockerfile.perl-5.40 index fe2c3318d..b0333fe37 100644 --- a/pkg/docker/Dockerfile.perl-5.40 +++ b/pkg/docker/Dockerfile.perl-5.40 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.perl-5.42 b/pkg/docker/Dockerfile.perl-5.42 index 04102f313..4fa2b6063 100644 --- a/pkg/docker/Dockerfile.perl-5.42 +++ b/pkg/docker/Dockerfile.perl-5.42 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.php-8.3 b/pkg/docker/Dockerfile.php-8.3 index 7d1962cce..f80513dde 100644 --- a/pkg/docker/Dockerfile.php-8.3 +++ b/pkg/docker/Dockerfile.php-8.3 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.php-8.4 b/pkg/docker/Dockerfile.php-8.4 index c8c63e239..ba5405ed0 100644 --- a/pkg/docker/Dockerfile.php-8.4 +++ b/pkg/docker/Dockerfile.php-8.4 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.php-8.5 b/pkg/docker/Dockerfile.php-8.5 index c8e812747..6fe01913f 100644 --- a/pkg/docker/Dockerfile.php-8.5 +++ b/pkg/docker/Dockerfile.php-8.5 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.python-3.12 b/pkg/docker/Dockerfile.python-3.12 index dc124fa63..6d8a451f0 100644 --- a/pkg/docker/Dockerfile.python-3.12 +++ b/pkg/docker/Dockerfile.python-3.12 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.python-3.12-slim b/pkg/docker/Dockerfile.python-3.12-slim index b49a68774..c5ce4f34d 100644 --- a/pkg/docker/Dockerfile.python-3.12-slim +++ b/pkg/docker/Dockerfile.python-3.12-slim @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.python-3.13 b/pkg/docker/Dockerfile.python-3.13 index a5d52b3f7..6d94e94ed 100644 --- a/pkg/docker/Dockerfile.python-3.13 +++ b/pkg/docker/Dockerfile.python-3.13 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.python-3.13-slim b/pkg/docker/Dockerfile.python-3.13-slim index 9f360f326..3a037f41b 100644 --- a/pkg/docker/Dockerfile.python-3.13-slim +++ b/pkg/docker/Dockerfile.python-3.13-slim @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.python-3.14 b/pkg/docker/Dockerfile.python-3.14 index 5ac2e7b82..1d4fe0597 100644 --- a/pkg/docker/Dockerfile.python-3.14 +++ b/pkg/docker/Dockerfile.python-3.14 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.python-3.14-slim b/pkg/docker/Dockerfile.python-3.14-slim index 93fdf5a42..1b3a8b3e8 100644 --- a/pkg/docker/Dockerfile.python-3.14-slim +++ b/pkg/docker/Dockerfile.python-3.14-slim @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.ruby-3.3 b/pkg/docker/Dockerfile.ruby-3.3 index b8fe61c47..8cd08db71 100644 --- a/pkg/docker/Dockerfile.ruby-3.3 +++ b/pkg/docker/Dockerfile.ruby-3.3 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.ruby-3.4 b/pkg/docker/Dockerfile.ruby-3.4 index 96139eba4..5d0e5dc9c 100644 --- a/pkg/docker/Dockerfile.ruby-3.4 +++ b/pkg/docker/Dockerfile.ruby-3.4 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.ruby-4.0 b/pkg/docker/Dockerfile.ruby-4.0 index 8f64b4fab..b7c3ec38c 100644 --- a/pkg/docker/Dockerfile.ruby-4.0 +++ b/pkg/docker/Dockerfile.ruby-4.0 @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/Dockerfile.wasm b/pkg/docker/Dockerfile.wasm index 83b35b516..1059c33d3 100644 --- a/pkg/docker/Dockerfile.wasm +++ b/pkg/docker/Dockerfile.wasm @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/pkg/docker/template.Dockerfile b/pkg/docker/template.Dockerfile index 1d9355393..f80d1300b 100644 --- a/pkg/docker/template.Dockerfile +++ b/pkg/docker/template.Dockerfile @@ -13,7 +13,7 @@ RUN set -ex \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ From c868c510bc7a3e8609f0b9f03357196175c025a3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 10:59:33 +0200 Subject: [PATCH 345/391] wasm: update Wasmtime to 48.0.5 Wasmtime 47.0.4 is affected by twelve advisories published on 2026-09-24 and 2026-10-02. The 47 line has no fix. This commit moves both Wasmtime pins to 48.0.5: the C API tarball in pkg/contrib, and the wasmtime, wasmtime-wasi and wasmtime-wasi-http crates of the wasm-wasi-component module. cargo audit on the old component lock reports exactly these twelve. OSV reports none for 48.0.5. Fixed in 48.0.3: GHSA-m63x-6p34-q65x, GHSA-jqpg-j7w6-42pr, GHSA-c9gc-w9vx-w86p, GHSA-j2g9-4prp-pf6h. Fixed in 48.0.4: GHSA-32h6-97mm-8q3c, GHSA-w4qr-p94g-mjhv, GHSA-gqmc-89g8-p25r, GHSA-j366-h8gg-77pm, GHSA-96f6-r43r-8c24, GHSA-mr2v-56j5-cmfc, GHSA-cfhf-m2cr-62wj, GHSA-hw8m-q44c-ggrf. Wasmtime 48 is a long-term support line. Upstream supports it for 24 months and backports security fixes to it. Wasmtime 49 is supported for 2 months and needs Rust 1.96.0. 48.0.5 has the code of 48.0.4. It only repairs the release publishing; upstream made no GitHub release for 48.0.4. By code reading, four of the twelve can be reached in the Unit build. Each needs a hostile guest. - GHSA-hw8m-q44c-ggrf: the wasm module, because the C API enables GC and exceptions. - GHSA-96f6-r43r-8c24: the wasm module, when access.filesystem is set. - GHSA-c9gc-w9vx-w86p: the component module, through any outgoing body. - GHSA-j2g9-4prp-pf6h: the component module, when access.filesystem is set. The other eight are not reached. The C API library is built with component-model-async, the cmake default, but the wasm module compiles only core modules with wasmtime_module_new(). The component module is built without component-model-async, gc, fuel, wasi-http sending and WASI p3. Wasmtime 48 changes three APIs that Unit uses. - The C API function wasi_config_preopen_dir() takes a bool fs_mutable in place of the directory and file permission masks. - The wasmtime-wasi crate has FsPerms in place of DirPerms and FilePerms. - The wasmtime-wasi-http hooks have a new send_request() signature and a new module path. Unit preopens every directory for read and write, so both preopen calls pass the read-write value. Guest outbound HTTP is still denied with HttpRequestDenied. Wasmtime 48 also denies TCP and UDP socket creation by default, and Unit does not allow it. A guest now gets the denial at create-tcp-socket instead of at connect or bind; both versions deny every bind and connect. No other C API symbol that the wasm module calls changed. Between 47.0.4 and 48.0.5 the C headers differ only in wasi.h, wasmtime/wasi.hh and the version macros. Wasmtime 48 needs Rust 1.95.0. The previous commit moves the Docker images to it. The lock audit allowlist drops crossbeam-epoch 0.9.18, because 48.0.5 uses 0.9.20. The chacha20 0.10.0 entry stays. EOL.md and pkg/eol.json name 48.0.5. https://github.com/freeunitorg/packages.freeunitorg.github.io/pull/12 adds the tarball to the mirror. The SHA-512 comes from the GitHub release asset, whose SHA-256 matches the digest that GitHub shows. Built and tested on pro4s-2 with Rust 1.95.0: test/test_wasm.py, test/test_wasm-wasi-component.py and test/test_app_lifecycle.py give 23 passed and 35 skipped (lifecycle cases for languages not built). Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- CHANGES | 9 + EOL.md | 4 +- docs/changes.xml | 12 ++ pkg/contrib/src/wasmtime/SHA512SUMS | 2 +- pkg/contrib/src/wasmtime/version | 2 +- pkg/contrib/wasmtime-lock-audit.allow.json | 9 +- pkg/eol.json | 6 +- src/wasm-wasi-component/Cargo.lock | 217 +++++++++++---------- src/wasm-wasi-component/Cargo.toml | 6 +- src/wasm-wasi-component/src/lib.rs | 41 ++-- src/wasm/nxt_rt_wasmtime.c | 4 +- 11 files changed, 168 insertions(+), 144 deletions(-) diff --git a/CHANGES b/CHANGES index 82e05ba8c..1fce0af4d 100644 --- a/CHANGES +++ b/CHANGES @@ -838,6 +838,15 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 which were exported symbols. The hooks now go only into the test programs. A build without "--tests" is unchanged. + *) Change: update wasmtime to 48.0.5. The WebAssembly module moves from + 43.0.1, and the WebAssembly component module from 47.0.2. Version 48 + is a long-term support release of wasmtime. It fixes twelve wasmtime + security advisories published on 24 September and 2 October 2026. + Each one needs a hostile WebAssembly guest. With the features that + FreeUnit builds, such a guest could panic the host, exhaust host + memory, read uninitialized host memory, or corrupt the GC heap. + Building the two modules now needs Rust 1.95.0 or newer. + Changes with FreeUnit 1.36.1 28 Aug 2026 *) Feature: configuration validation errors now say where they are. A diff --git a/EOL.md b/EOL.md index 622d88a49..0b2f159a2 100644 --- a/EOL.md +++ b/EOL.md @@ -92,8 +92,8 @@ needs an upgrade or replacement decision instead. | Eclipse Jetty | `jetty-util` / `jetty-server` / `jetty-http` jars bundled by the Java module | 9.4.58.v20250814 | **Aug 2025 (EOL)** † | Jetty 9.4 lost community support on 2025-08-14 and the bundled build is its last release. Needs an upgrade-or-replace decision — Jetty 10/11 are EOL too; 12.x is the supported line. | | Eclipse ECJ (JDT batch compiler) | JSP compilation jar bundled by the Java module | 3.26.0 | none published | Tracks Eclipse releases; endoflife.date has no product for it. Pinned build is from Jun 2021; current is 3.42.0 (Jun 2025). | | ClassGraph | classpath scanning jar bundled by the Java module | latest | — | No upstream EOL schedule; pinned and bumped as needed. | -| Wasmtime (C API) | WebAssembly runtime built by `pkg/contrib` and linked by the `wasm` module | 47.0.4 | none published | Fetched as a source tarball (`pkg/contrib/src/wasmtime/version`), so `cargo audit` never sees it; the `Audit (cargo)` pins job queries OSV for the pinned version instead. Same 47.0.4 as the Rust pin below; the 43.0.1 bump closed RUSTSEC-2026-0269, -0222, -0114 and three wasmtime-wasi path/permission advisories — see #400. | -| Wasmtime (Rust crate) | WebAssembly runtime used by the `wasm-wasi-component` module | 47.0.4 | none published | `src/wasm-wasi-component/Cargo.lock`, audited by the `Audit (cargo)` workflow on every lock. Bumped with the advisories it closes. | +| Wasmtime (C API) | WebAssembly runtime built by `pkg/contrib` and linked by the `wasm` module | 48.0.5 | none published | Fetched as a source tarball (`pkg/contrib/src/wasmtime/version`), so `cargo audit` never sees it; the `Audit (cargo)` pins job queries OSV for the pinned version instead. Same 48.0.5 as the Rust pin below; 48 is a long-term support line. The 47.0.4 to 48.0.5 bump closed twelve advisories from 2026-09-24 and 2026-10-02. The 43.0.1 bump closed RUSTSEC-2026-0269, -0222, -0114 and three wasmtime-wasi path/permission advisories — see #400. | +| Wasmtime (Rust crate) | WebAssembly runtime used by the `wasm-wasi-component` module | 48.0.5 | none published | `src/wasm-wasi-component/Cargo.lock`, audited by the `Audit (cargo)` workflow on every lock. Bumped with the advisories it closes. | ## Rules diff --git a/docs/changes.xml b/docs/changes.xml index f60351347..64db468c0 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -1124,6 +1124,18 @@ A build without "--tests" is unchanged. + + +update wasmtime to 48.0.5. The WebAssembly module moves from 43.0.1, and the +WebAssembly component module from 47.0.2. Version 48 is a long-term support +release of wasmtime. It fixes twelve wasmtime security advisories published on +24 September and 2 October 2026. Each one needs a hostile WebAssembly guest. +With the features that FreeUnit builds, such a guest could panic the host, +exhaust host memory, read uninitialized host memory, or corrupt the GC heap. +Building the two modules now needs Rust 1.95.0 or newer. + + + diff --git a/pkg/contrib/src/wasmtime/SHA512SUMS b/pkg/contrib/src/wasmtime/SHA512SUMS index 4a150cf06..d462e4761 100644 --- a/pkg/contrib/src/wasmtime/SHA512SUMS +++ b/pkg/contrib/src/wasmtime/SHA512SUMS @@ -1 +1 @@ -3a27dada5ed2263f4ce769cd81dc8d7d5cb79ec3ccf02bd745a8231302dae04e5f6d58f38d4c0a3201dd4f9022efcf247548f62d0be7baedcb128af29b40b078 wasmtime-v47.0.4-src.tar.gz \ No newline at end of file +a354ccf737a0727d8c83944ca5d909da53cc95637646cd80e0a7c70d54ffc809531c3ef96bcd4bdc40754f20dfefe6c1d4027f0c77b608e3b2699d95b53a2baa wasmtime-v48.0.5-src.tar.gz \ No newline at end of file diff --git a/pkg/contrib/src/wasmtime/version b/pkg/contrib/src/wasmtime/version index dd0483ab4..631363286 100644 --- a/pkg/contrib/src/wasmtime/version +++ b/pkg/contrib/src/wasmtime/version @@ -1 +1 @@ -WASMTIME_VERSION := 47.0.4 +WASMTIME_VERSION := 48.0.5 diff --git a/pkg/contrib/wasmtime-lock-audit.allow.json b/pkg/contrib/wasmtime-lock-audit.allow.json index f338f7c16..b73c602c1 100644 --- a/pkg/contrib/wasmtime-lock-audit.allow.json +++ b/pkg/contrib/wasmtime-lock-audit.allow.json @@ -8,19 +8,12 @@ "reviewed again rather than carrying an acceptance forward silently." ], "allow": [ - { - "kind": "vulnerability", - "id": "RUSTSEC-2026-0204", - "package": "crossbeam-epoch", - "version": "0.9.18", - "reason": "Linked through rayon, which wasmtime pulls in for the parallel-compilation feature. The bug is an invalid pointer dereference in the fmt::Pointer impl of Atomic and Shared; neither wasmtime nor Unit formats an epoch pointer, so no call site reaches it. The archive is vendored and cannot be relocked, and wasmtime 48.x already resolves crossbeam-epoch 0.9.20, so the next pin bump clears this." - }, { "kind": "yanked", "id": null, "package": "chacha20", "version": "0.10.0", - "reason": "Linked through rand, which wasmtime-wasi uses to seed guest randomness. A yank is not an advisory: no RustSec record applies to this version. 0.10.1 is yanked as well and 0.10.2 is clean, but the archive is vendored, so the version moves only when the pin does." + "reason": "Linked through rand 0.10.1, which wasmtime-wasi uses to seed guest randomness. A yank is not an advisory: no RustSec record applies to this version. 0.10.1 is yanked as well and 0.10.2 is clean, but the Wasmtime 48.0.5 archive is vendored and its lock still names 0.10.0, so the version moves only when the pin does." } ] } diff --git a/pkg/eol.json b/pkg/eol.json index 0e94bfe15..b3aa883d7 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -129,14 +129,14 @@ }, "wasmtime_c_api": { "role": "WebAssembly runtime built by pkg/contrib and linked by the wasm module (pkg/contrib/src/wasmtime/version, WASMTIME_VERSION)", - "version": "47.0.4", + "version": "48.0.5", "eol": null, - "eol_note": "Fetched as a source tarball from packages.freeunit.org, so its Cargo.lock is not among the repository locks. The Audit (cargo) pins job verifies the tarball digest, queries OSV for the direct source-package pin, and audits the archived Cargo.lock, so an advisory here arrives as a red job rather than silence. That lock is the whole Wasmtime workspace, so pkg/contrib/check-lock-audit.py fails it only for a crate the C API library links, with pkg/contrib/wasmtime-lock-audit.allow.json recording the accepted ones; the archive is vendored, so a finding moves only when the pin does. Same 47.0.4 as the wasmtime_crate pin; the 43.0.1 bump closed RUSTSEC-2026-0269 (filesystem sandbox escape), -0222, -0114, and wasmtime-wasi -0149, -0182 and -0188 (WASI path and file-permission bypasses). Tracked in freeunitorg/freeunit#400.", + "eol_note": "Fetched as a source tarball from packages.freeunit.org, so its Cargo.lock is not among the repository locks. The Audit (cargo) pins job verifies the tarball digest, queries OSV for the direct source-package pin, and audits the archived Cargo.lock, so an advisory here arrives as a red job rather than silence. That lock is the whole Wasmtime workspace, so pkg/contrib/check-lock-audit.py fails it only for a crate the C API library links, with pkg/contrib/wasmtime-lock-audit.allow.json recording the accepted ones; the archive is vendored, so a finding moves only when the pin does. Same 48.0.5 as the wasmtime_crate pin. Wasmtime 48 is a long-term support line: upstream supports a release whose number divides by 12 for 24 months and backports security fixes to it. The 47.0.4 to 48.0.5 bump closed twelve advisories published on 2026-09-24 and 2026-10-02. The 43.0.1 bump closed RUSTSEC-2026-0269 (filesystem sandbox escape), -0222, -0114, and wasmtime-wasi -0149, -0182 and -0188 (WASI path and file-permission bypasses). Tracked in freeunitorg/freeunit#400.", "source": "https://github.com/bytecodealliance/wasmtime/releases" }, "wasmtime_crate": { "role": "WebAssembly runtime used by the wasm-wasi-component module (src/wasm-wasi-component/Cargo.lock)", - "version": "47.0.4", + "version": "48.0.5", "eol": null, "eol_note": "Audited by the Audit (cargo) workflow, which checks every tracked Cargo.lock against the RustSec database, so an advisory here arrives as a red job rather than silence. Bumped with the advisories it closes.", "source": "https://github.com/bytecodealliance/wasmtime/releases" diff --git a/src/wasm-wasi-component/Cargo.lock b/src/wasm-wasi-component/Cargo.lock index b7b2598cc..72dd6f7a0 100644 --- a/src/wasm-wasi-component/Cargo.lock +++ b/src/wasm-wasi-component/Cargo.lock @@ -111,18 +111,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" -[[package]] -name = "cap-fs-ext" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ff379b70af8e08307a8f65e7040c7301cb4a572538ade16b4984f0da77847f" -dependencies = [ - "cap-primitives", - "cap-std", - "io-lifetimes 3.0.1", - "windows-sys 0.61.2", -] - [[package]] name = "cap-primitives" version = "4.0.3" @@ -141,18 +129,6 @@ dependencies = [ "winx", ] -[[package]] -name = "cap-std" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8" -dependencies = [ - "cap-primitives", - "io-extras", - "io-lifetimes 3.0.1", - "rustix", -] - [[package]] name = "cc" version = "1.2.66" @@ -238,27 +214,27 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16c273ba1bfc3a1cb27cb4f83df27134f3cb638a61f5ed79ab22cf86eb13da0d" +checksum = "3bfbb8f30b72ef5cedf2dbd85ee2f167a33c888d492a40e999250187138663c9" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf8909326a466739e83ffe11e74c0c8c6b4bfa911b612aece915daa746cff58" +checksum = "5715fdd9d897798ba2bb834df5d6c1129319d51e82ddcf81812c23231e9b852b" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb26b06b54d8b2f8cdf4d440a32fc3b3b91c71c162333a69a70a66fa265fc45" +checksum = "1a266da98b248dd98dfc8a46d45d57810db04a55b81a6d8c7a9c6cf0bd2db7c3" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -266,9 +242,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce4e57dbb7f73d08011808a19787c3a03c3e7854b2a479ed0b788915fb671be" +checksum = "c67c1887ed2194ef3ecc283a9add4fe8a79d249a293063046f2adedc31ae7200" dependencies = [ "serde", "serde_derive", @@ -277,9 +253,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b0a8968f33d0bc13bc86fe70dd3947e93acc9f358ed9848e9988448972a9be" +checksum = "90103abeafa22cc578c577999bdf1b51c32b8a59e15a09789841b4e9a854588d" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -308,9 +284,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f69e8e2fe9deb48ef7b8d0c61612f8c1e2277d0421c201e30a4c844a4ebac698" +checksum = "d73eaff808bc302e2899e42d822c0e64df23f38facb98ecdf695d1817d2063a3" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -321,24 +297,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad4db1e87a65e9c5a832e3ede160830d4e34df938ac38b5604819f8825687e73" +checksum = "de19c4fc55ac3cdeb8e153f35ef9f51db197da91a64c9811d0ddd21ed1b9d18d" [[package]] name = "cranelift-control" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9248cd37bb6ec460981f57ead368e32b1eca7a207ad50e3e9c7adc3b161f20e9" +checksum = "9bbd919579d74a7212e6e849dc24ff015cb823e70051d01559eea828efaf773f" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b8bc91c0a3530d132acbf118965dabe9a084029eb0fa74b64760360dec3316" +checksum = "af201af93fa8a576ac4b04b901ac3f252545d7b703053c922526fce28f65a7be" dependencies = [ "cranelift-bitset", "serde", @@ -348,9 +324,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5086f2ebc084a98b387e522680f62ef65a512444d8382151e03088716d3a1714" +checksum = "69395ceee43706b93bf869b3a6631ab8d9dcb2bf19f9af4d1d2db0cb9e18af18" dependencies = [ "cranelift-codegen", "hashbrown 0.17.1", @@ -361,15 +337,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a658670f779afc2df083be201ebe54805f69b7b9ffa8dab04142b72ed751c9" +checksum = "8e8d98a81f71e2ddc61849ca1454ff424fbf42a3ee5529365be8c0859ddf5b9a" [[package]] name = "cranelift-native" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a41083b1fd953debd6f32d21fcda765b258f323ce907fdd6b4715f2769c478" +checksum = "273c79e0265b754f67ff76c42d6db4856dabf9101e3bb1b23c83255e2438dd02" dependencies = [ "cranelift-codegen", "libc", @@ -378,9 +354,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.134.4" +version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244d9cd0759b0a3ccdd587cafb13434f9ea43b5c9d07701d277daf5cd0f7b7c1" +checksum = "2b73d7e686f3518831f88ffe88be02c6c92be09930fc5ab90a3dfd9a4de3815e" [[package]] name = "crc32fast" @@ -1091,9 +1067,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bd641cb6a3ec5bffe60405568933163e0ea619da0943e9d2a417ceaa7fe82" +checksum = "dfc9a7073f2530aab972ac0a7ce43624ff59de8ad06b76a4ef4021f7ddd1035a" dependencies = [ "cranelift-bitset", "log", @@ -1103,9 +1079,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd4024d1e559eaf5579bc8ad84a2b5915f34b47acc13a06ffba7dac401b166" +checksum = "864f92c9538de48325a4f8709ec83e56471446798d48a29073ca094c5447b549" dependencies = [ "proc-macro2", "quote", @@ -1534,14 +1510,26 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasm-encoder" -version = "0.252.0" +version = "0.254.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8185ae345fa5687c054626ff9a50e7089797a343d9904d1dc9820eb4c4d3196f" +checksum = "ebaca0b540bdb15d195ff34fb23074255948708b8916a11a339ba4513dc61906" dependencies = [ "leb128fmt", "wasmparser", ] +[[package]] +name = "wasm-metadata" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1880dd9a848aa68f387966256c34ad49abb4607e3cfa35720d2723d0591cd3" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-wasi-component" version = "0.1.0" @@ -1564,9 +1552,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.252.0" +version = "0.254.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3eb099dcadcde5be9eef55e3a337128efd4e44b4c93122487e4d2e4e1c6627c" +checksum = "c5ff8685b587722894b6f25958fc608d315b93b8d23647bfb567664e7006d757" dependencies = [ "bitflags", "hashbrown 0.17.1", @@ -1577,9 +1565,9 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.252.0" +version = "0.254.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7142797de29b35ab8dbf15c00f55fda75d409da4c423a8ab8bd6b667a785824b" +checksum = "8b03c8f270f3f384f70f42869b5ff7f6253fe4f0aa8590f7edd332c1943e5f51" dependencies = [ "anyhow", "termcolor", @@ -1588,16 +1576,15 @@ dependencies = [ [[package]] name = "wasmtime" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf5b8af5713146e3a62d940c71e3eb2ad1ebacfa6b55f0c8509a5b4b87718" +checksum = "b94eda6b1c1d8ade526a67e4ce50cc6234d1c7af68e5d7bf13f40bc07e70daba" dependencies = [ "addr2line", "async-trait", "bitflags", "bumpalo", "cc", - "cfg-if", "encoding_rs", "futures", "libc", @@ -1630,9 +1617,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58793271eab7ec26bab99ca497131f8b37702e7dc39e2f2d18293e3e915b4964" +checksum = "84ca4865ece3a37abb34c2df96c741ff4aed308d35744d84095cff79848752d4" dependencies = [ "anyhow", "cpp_demangle", @@ -1661,9 +1648,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5bba4eff4804620bae04d7793308d97a0c5a8a97ea19e635452518f84560f84" +checksum = "5ae080b05e5e0356a2167c478c264e26f3fe5dbaaf1b285f8f5501301b99facb" dependencies = [ "anyhow", "proc-macro2", @@ -1676,15 +1663,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d7032ff6b4d7a45e05be1cb32ec1756df5f2669f08296dbe2c61c3789dead2c" +checksum = "e8f014d9ad6f6acbc449e9d7c4e7b7757b4b6b1a84a3662f66ea6da2ebed788c" [[package]] name = "wasmtime-internal-core" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41d3b2cb9c3de7b696690af070d408b5e357011832f8b8bb7fbb315dbb620ed" +checksum = "ecc0a0310fc4a3f545427c7880362657f72f838ffcfe4d3b8c864021df3b3488" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -1694,11 +1681,10 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5cda5b90247978dd2ba6ca4e309508da19e9244445671d2d0e6a34e5c3b847" +checksum = "cb6c8d2b6f8ed3a784fcd127a1936589db8051febbd228230660175e41b5751d" dependencies = [ - "cfg-if", "cranelift-codegen", "cranelift-control", "cranelift-entity", @@ -1721,12 +1707,11 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d126366176553e8f304e8ac2078a4cc37134fe8f19d4d4695a09c5a2f577f10" +checksum = "92b558b5892ca405508103641fa3a3ec2e986b1cc679a083ed47e6bcb08ecf31" dependencies = [ "cc", - "cfg-if", "libc", "rustix", "wasmtime-environ", @@ -1736,9 +1721,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1d28ff131bc40e11d84bc3a9fa0d264506439ffbbc89f3562fbb529d1caa617" +checksum = "c0669ce70a8a8c92eb8ad148c1d7b9b771aab37787e9462efd75b2e2b4575406" dependencies = [ "cc", "wasmtime-internal-versioned-export-macros", @@ -1746,11 +1731,10 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a52de8b6afbfcd618073f592c1450cb661574e7061f3f4a44f1f7ec0c7f7909" +checksum = "6aa66405e95fc7b82adb195c14fa8b38f45046c3ad99a273b06262b8eb755e76" dependencies = [ - "cfg-if", "libc", "wasmtime-internal-core", "windows-sys 0.61.2", @@ -1758,11 +1742,10 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5524fbaaf3d2134dd6229164d7fc81e443cce06d1e0576524d5f4e86e747cb13" +checksum = "70555458e519d6fb8790e042b4274132ab58171ae0f2a6917c3dc29ded0184d0" dependencies = [ - "cfg-if", "cranelift-codegen", "log", "object", @@ -1771,9 +1754,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93bb9ae112a80618510f938275a5d88637b656e7d0a021fc906f52206c708ad" +checksum = "fca407e3e2809c0e6568be132065095a16f184c30c6bee4157dfdc497aee10a1" dependencies = [ "proc-macro2", "quote", @@ -1782,33 +1765,32 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdbdd9d6fab1ecb67a7dc189e32c9e341444945fb709437bf8aab25d9a88459" +checksum = "2aacdb42fb0b393353e402a1534e0b492a6b0fddf6528bfaec65c5f56b990c43" dependencies = [ "anyhow", "bitflags", "heck", "indexmap", + "wit-component", "wit-parser", ] [[package]] name = "wasmtime-wasi" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b373095a6dc8382da56882ad16d7ea16771dd6927dd480b1ce3fe7c4b816d2cf" +checksum = "1803e99b97392e59866a9f3178ed38f8addb74412facbcb6dc102b2e7420da76" dependencies = [ "async-trait", "bitflags", "bytes", - "cap-fs-ext", - "cap-std", - "cfg-if", + "cap-primitives", "futures", - "io-lifetimes 3.0.1", "rand", "rustix", + "rustix-linux-procfs", "thiserror 2.0.18", "tokio", "tracing", @@ -1817,13 +1799,14 @@ dependencies = [ "wasmtime-wasi-io", "wiggle", "windows-sys 0.61.2", + "winx", ] [[package]] name = "wasmtime-wasi-http" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372fc7845f26cd8f69bcbf6e4234fcc032f718480c7722fa20a28eb74d58998e" +checksum = "1aeecf39d9003a7f3199ae7b9e70ad9115b3d73746ff9dce87a500fd95d572ea" dependencies = [ "async-trait", "bytes", @@ -1832,6 +1815,7 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "pin-project-lite", "tokio", "tracing", "wasmtime", @@ -1841,9 +1825,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-io" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b3c9b39b1a1b3029f80e614623e182f165133dfaca8610f156450343f080a4" +checksum = "1af4069fb7fc9cdc72410be96fb7f83b772e500cf4d0758d7fe6b55cefb303b8" dependencies = [ "async-trait", "bytes", @@ -1863,9 +1847,9 @@ dependencies = [ [[package]] name = "wiggle" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08bd1d66a10c7d4deaeb746cb1c5b5c62fb631907f18dfe058ecba71c99002" +checksum = "314611904412397366e044ace02f657dc69db7fb419c630a319a843fe65afee4" dependencies = [ "bitflags", "thiserror 2.0.18", @@ -1877,9 +1861,9 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19eb81a4ceda41cc6fa2adc8e865473fc2e85630149ed7f5d95356b0010f0ed3" +checksum = "d0c1d1d148dd7a0425eef52a5bc9e26bd35ebfe8a5f0fcfc5480a6d3cee56109" dependencies = [ "heck", "proc-macro2", @@ -1891,9 +1875,9 @@ dependencies = [ [[package]] name = "wiggle-macro" -version = "47.0.4" +version = "48.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0e0d7c6d28846ffa438e421976734a2057f13a7e20af9f2a8607a6714e1e358" +checksum = "488783149d44017c9807189b16cc09e02e84059740fe64178b5ab3cc19e9fc0b" dependencies = [ "proc-macro2", "quote", @@ -2082,11 +2066,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "wit-component" +version = "0.254.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c96b3cea6838dbea55173baa56991fcb087896c321b0a770db201e5a0bd3506" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + [[package]] name = "wit-parser" -version = "0.252.0" +version = "0.254.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4266bea110371c620ccf3201c5023676046bc4556e5c7cfb5d500bda5ebc162d" +checksum = "22672d61bfb9df46a16b46ce38522293aee1cc4749fc84a3464ce4083b3d5fd6" dependencies = [ "anyhow", "hashbrown 0.17.1", diff --git a/src/wasm-wasi-component/Cargo.toml b/src/wasm-wasi-component/Cargo.toml index 1bde3e10e..f542d0fe6 100644 --- a/src/wasm-wasi-component/Cargo.toml +++ b/src/wasm-wasi-component/Cargo.toml @@ -17,14 +17,14 @@ http-body = { version = "1.0.0", default-features = false } http-body-util = "0.1.0" hyper = "1.4.1" tokio = { version = "1.33.0", default-features = false } -wasmtime = { version = "47.0.2", default-features = false, features = ['anyhow', 'component-model', 'cranelift'] } -wasmtime-wasi = "47.0.2" +wasmtime = { version = "48.0.5", default-features = false, features = ['anyhow', 'component-model', 'cranelift'] } +wasmtime-wasi = "48.0.5" # default-features = false drops the "default-send-request" feature and # with it the whole rustls TLS stack (rustls, tokio-rustls, webpki-roots, # rustls-webpki) that exists only so a guest can dial out. "p2" is in the # crate's default set too, so it has to be named explicitly or the # wasmtime_wasi_http::p2 module this file uses disappears with it. -wasmtime-wasi-http = { version = "47.0.2", default-features = false, features = ['p2'] } +wasmtime-wasi-http = { version = "48.0.5", default-features = false, features = ['p2'] } [build-dependencies] bindgen = "0.72" diff --git a/src/wasm-wasi-component/src/lib.rs b/src/wasm-wasi-component/src/lib.rs index 6c4bce706..65b8ed455 100644 --- a/src/wasm-wasi-component/src/lib.rs +++ b/src/wasm-wasi-component/src/lib.rs @@ -5,6 +5,7 @@ use http_body_util::{BodyExt, Full}; use hyper::Error; use std::borrow::Cow; use std::ffi::{CStr, CString}; +use std::future::Future; use std::mem::MaybeUninit; use std::os::raw::c_int; use std::process::exit; @@ -15,16 +16,14 @@ use tokio::sync::mpsc; use wasmtime::component::{Component, Linker, ResourceTable}; use wasmtime::{Config, Engine, Store}; use wasmtime_wasi::p2::add_to_linker_async; -use wasmtime_wasi::{DirPerms, FilePerms, WasiCtx, WasiCtxBuilder, - WasiCtxView, WasiView}; -use wasmtime_wasi_http::p2::bindings::http::types::{ErrorCode, Scheme}; +use wasmtime_wasi::{FsPerms, WasiCtx, WasiCtxBuilder, WasiCtxView, + WasiView}; +use wasmtime_wasi_http::p2::bindings::http::types::Scheme; use wasmtime_wasi_http::p2::bindings::ProxyPre; use wasmtime_wasi_http::p2::body::HyperOutgoingBody; -use wasmtime_wasi_http::p2::types::{HostFutureIncomingResponse, - OutgoingRequestConfig}; -use wasmtime_wasi_http::p2::{HttpResult, WasiHttpCtxView, WasiHttpHooks, - WasiHttpView}; -use wasmtime_wasi_http::WasiHttpCtx; +use wasmtime_wasi_http::{Error as HttpError, RequestOptions, WasiBody, + WasiHttpCtx, WasiHttpCtxView, WasiHttpHooks, + WasiHttpView}; #[allow( non_camel_case_types, @@ -353,12 +352,7 @@ impl GlobalState { cx.inherit_stderr(); cx.inherit_env(); for dir in self.global_config.dirs.iter() { - cx.preopened_dir( - dir, - dir, - DirPerms::all(), - FilePerms::all(), - )?; + cx.preopened_dir(dir, dir, FsPerms::ReadWrite)?; } cx.build() }, @@ -973,10 +967,21 @@ struct DenyOutboundHttp; impl WasiHttpHooks for DenyOutboundHttp { fn send_request( &mut self, - _request: hyper::Request, - _config: OutgoingRequestConfig, - ) -> HttpResult { - Err(ErrorCode::HttpRequestDenied.into()) + _request: hyper::Request, + _options: Option, + _fut: Box> + Send>, + ) -> Box< + dyn Future< + Output = Result< + ( + hyper::Response, + Box> + Send>, + ), + HttpError, + >, + > + Send, + > { + Box::new(async { Err(HttpError::HttpRequestDenied) }) } } diff --git a/src/wasm/nxt_rt_wasmtime.c b/src/wasm/nxt_rt_wasmtime.c index a1945d2cc..570a6298a 100644 --- a/src/wasm/nxt_rt_wasmtime.c +++ b/src/wasm/nxt_rt_wasmtime.c @@ -281,7 +281,9 @@ nxt_wasmtime_wasi_init(const nxt_wasm_ctx_t *ctx) wasi_config_inherit_stderr(wasi_config); for (dir = ctx->dirs; dir != NULL && *dir != NULL; dir++) { -#if defined(WASMTIME_VERSION_MAJOR) && (WASMTIME_VERSION_MAJOR >= 27) +#if defined(WASMTIME_VERSION_MAJOR) && (WASMTIME_VERSION_MAJOR >= 48) + wasi_config_preopen_dir(wasi_config, *dir, *dir, true); +#elif defined(WASMTIME_VERSION_MAJOR) && (WASMTIME_VERSION_MAJOR >= 27) wasi_config_preopen_dir(wasi_config, *dir, *dir, WASMTIME_WASI_DIR_PERMS_READ|WASMTIME_WASI_DIR_PERMS_WRITE, WASMTIME_WASI_FILE_PERMS_READ|WASMTIME_WASI_FILE_PERMS_WRITE); From d4d7f559c905b906c584690a2f3cc3794d44039a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 06:31:15 +0200 Subject: [PATCH 346/391] isolation: create mount destinations without following symlinks The prototype ran mkdir_p on the mount destination before the openat2 check. mkdir follows symlinks, so a symlinked component inside the rootfs created directories outside it. Missing components are now created with mkdirat against a parent opened with openat2 RESOLVE_BENEATH. Each prefix is resolved from the rootfs fd, so relative in-tree links such as lib -> usr/lib still work. Absolute symlinks fail with EXDEV, as before. When openat2 returns EAGAIN (a rename or a mount on the system during a ".." lookup), it is tried again up to three times. An in-tree link whose target is missing now fails with ENOENT. The old code followed the link and created the target. The log names the component that cannot be resolved beneath the rootfs. Without openat2, each component is opened with O_NOFOLLOW, and a symlinked component is refused. This fallback is used when the header is missing, or when openat2 fails with ENOSYS (Linux older than 5.6), EPERM (a seccomp profile or gVisor) or EINVAL (an unknown resolve flag). The fallback is stricter, so it is safe in all these cases. The old code mounted on the unchecked path without openat2. A missing rootfs directory is refused with "open rootfs(...)", not created. A build with openat2 has refused it since 1.36.0. Older versions, and a build without openat2 such as on FreeBSD, created it with mkdir_p of the first destination. The rootfs path is operator config, so unitd does not create it as root. On Linux the mount targets the fd through /proc/self/fd, never the path. The FreeBSD nmount() path is unchanged. When a destination fails, the prototype now unmounts only the destinations that it mounted. The old code also unmounted, by path, the failed destination and the destinations it skipped (language_deps is off, or the host path is missing). These are not mounted, and their paths can go through a symlink. The first test makes "usr" and "opt" in the rootfs symlinks out of the rootfs. A distro Python has its stdlib under /usr. The toolcache Python in CI has it under /opt. The old code created directories behind the symlink. The new code refuses the app. The second test checks that the prototype does not unmount the refused destination. The third test checks that a missing rootfs is refused and not created. The first two tests load the app once with a plain rootfs. The mount table then shows which language_deps destinations start with "usr" or "opt". A test skips only when none does. Otherwise it requires the refusal record. The record must give the reason of the branch that unitd takes: "escapes rootfs or is not a directory" with openat2, "is a symlink or not a directory" without it. The fallback runs only when openat2 is missing or refused, and no CI kernel refuses it. A new C test, nxt_isolation_mount_dst_test, runs the walk on four destinations: a link out of the rootfs, the same link one level down, a link inside the rootfs, and a path with no link. It runs each one with openat2 and again with the fallback forced. A test hook, nxt_isolation_test_mount_dst_open(), forces the fallback. The hook is under NXT_TESTS, so only the test programs link it. The walk needs no privileges. The test runs in build/tests, in the c-tests job. Co-Authored-By: Claude Opus 5.5 Reviewed-by: DeepSeek V4.1 Flash Reviewed-by: Claude Fable 5.1 --- CHANGES | 10 + auto/sources | 1 + docs/changes.xml | 13 + src/nxt_isolation.c | 426 +++++++++++++++++------- src/nxt_isolation.h | 6 + src/test/nxt_isolation_mount_dst_test.c | 378 +++++++++++++++++++++ src/test/nxt_tests.c | 6 + src/test/nxt_tests.h | 3 + test/test_python_isolation.py | 166 +++++++++ 9 files changed, 891 insertions(+), 118 deletions(-) create mode 100644 src/test/nxt_isolation_mount_dst_test.c diff --git a/CHANGES b/CHANGES index 1fce0af4d..fcf203145 100644 --- a/CHANGES +++ b/CHANGES @@ -74,6 +74,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 now refuses such an offset, or an offset that is not aligned, and exits at startup. + *) Bugfix: an application with a rootfs created the missing + directories of each mount destination with mkdir(), which follows + symlinks. A symlink inside the rootfs could make unitd create empty + directories outside it. The directories are now created without + following symlinks. On a kernel without openat2() (older than 5.6) + a rootfs with a symlinked component in a mount destination (for + example usr -> real_usr) is now refused. A missing rootfs + directory is now refused on every system. Builds without + openat2(), such as on FreeBSD, created it before. + *) Security: the router and libunit read records from shared memory that the other process can still write, without enough checks. A buffer that was not a whole number of records was read past its diff --git a/auto/sources b/auto/sources index e61217186..aa7e92c64 100644 --- a/auto/sources +++ b/auto/sources @@ -239,6 +239,7 @@ fi if [ $NXT_HAVE_ROOTFS = YES ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_fs_mount.c" + NXT_TEST_SRCS="$NXT_TEST_SRCS src/test/nxt_isolation_mount_dst_test.c" fi diff --git a/docs/changes.xml b/docs/changes.xml index 64db468c0..a0f3e3f1d 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -113,6 +113,19 @@ of more than about 1,000 bytes can get a 500 response. + + +an application with a rootfs created the missing directories of each mount +destination with mkdir(), which follows symlinks. A symlink inside the rootfs +could make unitd create empty directories outside it. The directories are +now created without following symlinks. On a kernel without openat2() (older +than 5.6) a rootfs with a symlinked component in a mount destination (for +example usr -> real_usr) is now refused. A missing rootfs directory is now +refused on every system. Builds without openat2(), such as on FreeBSD, +created it before. + + + when unitd ran as root and the control socket was given to another user or diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index d5ea563f0..b9f3a9322 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -846,21 +846,263 @@ nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process) } +#if (NXT_HAVE_OPENAT2) +static nxt_bool_t nxt_isolation_no_openat2; +#endif + + +/* + * Open the mount destination rel, a path relative to rootfs_fd, as a + * directory. Create each missing component with mkdirat() against its + * parent fd. mkdirat() does not follow a symlink in the last component, + * so no directory is ever created outside the rootfs. With openat2() each + * prefix is resolved from rootfs_fd with RESOLVE_BENEATH: relative in-tree + * links such as "lib -> usr/lib" still work; absolute links and links that + * leave the rootfs fail with EXDEV. Without openat2() a symlinked + * component is refused. + * + * The returned O_PATH fd pins the directory inode, not its name: the + * directory can still be renamed after resolution. This is a known limit + * of fd walks. + */ + +static nxt_int_t +nxt_isolation_mount_dst_open(nxt_task_t *task, int rootfs_fd, + const u_char *dst, const u_char *rel, int *fdp) +{ + int fd, parent, flags; +#if (NXT_HAVE_OPENAT2) + int tries; +#endif + char *path, *p, *comp, save; + size_t len; + nxt_err_t err; +#if (NXT_HAVE_OPENAT2) + struct open_how how; +#endif + +#if defined(O_PATH) + flags = O_PATH | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC; +#else + flags = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC; +#endif + + /* + * A module path such as "//usr/lib" leaves a '/' at the start of rel. + * openat2() with RESOLVE_BENEATH refuses an absolute path. + */ + while (*rel == '/') { + rel++; + } + + len = nxt_strlen(rel); + + path = nxt_malloc(len + 1); + if (nxt_slow_path(path == NULL)) { + return NXT_ERROR; + } + + nxt_memcpy(path, rel, len + 1); + + parent = rootfs_fd; + p = path; + + for ( ;; ) { + while (*p == '/') { + p++; + } + + if (*p == '\0') { + break; + } + + comp = p; + + while (*p != '\0' && *p != '/') { + p++; + } + + save = *p; + *p = '\0'; + + /* + * A module path from the build (for example sys.path) can contain + * "." or "..". Without openat2() ".." would climb out of the rootfs. + */ + if (nxt_slow_path(nxt_strcmp(comp, ".") == 0 + || nxt_strcmp(comp, "..") == 0)) + { + nxt_alert(task, "mount destination %s: component \"%s\" " + "is not allowed", dst, comp); + goto fail; + } + + if (mkdirat(parent, comp, 0777) != 0 && nxt_errno != EEXIST) { + nxt_alert(task, "mount destination %s: mkdirat(\"%s\") %E", + dst, comp, nxt_errno); + goto fail; + } + + fd = -1; + +#if (NXT_HAVE_OPENAT2) + if (!nxt_isolation_no_openat2) { + nxt_memzero(&how, sizeof(how)); + how.flags = O_PATH | O_DIRECTORY | O_CLOEXEC; + how.resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS; + + /* + * Resolve the full prefix from rootfs_fd, not comp from parent. + * From parent, an in-tree link such as "usr/lib64 -> ../lib" + * fails. A rename race can only cause ENOENT, not an escape. + * + * With RESOLVE_BENEATH the kernel returns EAGAIN when a rename + * or a mount occurs on the system during a ".." lookup. Try + * again a small number of times. + */ + for (tries = 3; tries > 0; tries--) { + fd = syscall(SYS_openat2, rootfs_fd, path, &how, sizeof(how)); + + if (fd != -1 || nxt_errno != EAGAIN) { + break; + } + } + + if (fd == -1) { + err = nxt_errno; + + if (err == ENOENT) { + nxt_alert(task, "mount destination %s: component \"%s\" " + "cannot be resolved beneath rootfs", dst, comp); + goto fail; + } + + /* + * ENOSYS: openat2() is Linux 5.6+. EPERM: a seccomp + * profile or gVisor denies the syscall. EINVAL: the + * kernel does not know a resolve flag. The fallback + * below is stricter, so it is safe in all three cases. + */ + if (err != ENOSYS && err != EPERM && err != EINVAL) { + nxt_alert(task, "mount destination %s: component \"%s\" " + "escapes rootfs or is not a directory: %E", + dst, comp, err); + goto fail; + } + + nxt_isolation_no_openat2 = 1; + + /* Logged once per prototype start. */ + nxt_log(task, NXT_LOG_WARN, + "openat2() is not supported by the running kernel; " + "rootfs mount destinations must not contain " + "symlinks"); + } + } +#endif + + if (fd == -1) { + fd = openat(parent, comp, flags); + + if (fd == -1) { + err = nxt_errno; + + /* + * Linux returns ENOTDIR for a symlink opened with + * O_NOFOLLOW | O_DIRECTORY. POSIX allows ELOOP, and + * other systems can return it. + */ + if (err == ENOTDIR || err == ELOOP) { + nxt_alert(task, "mount destination %s: component \"%s\" " + "is a symlink or not a directory", dst, comp); + + } else { + nxt_alert(task, "mount destination %s: openat(\"%s\") %E", + dst, comp, err); + } + + goto fail; + } + } + + *p = save; + + if (parent != rootfs_fd) { + close(parent); + } + + parent = fd; + } + + nxt_free(path); + + if (nxt_slow_path(parent == rootfs_fd)) { + nxt_alert(task, "mount destination %s is the rootfs", dst); + return NXT_ERROR; + } + + *fdp = parent; + + return NXT_OK; + +fail: + + if (parent != rootfs_fd) { + close(parent); + } + + nxt_free(path); + + return NXT_ERROR; +} + + +#if (NXT_TESTS) + +/* + * Lets src/test/nxt_isolation_mount_dst_test.c run the walk above. When + * fallback is set, the walk does not call openat2(). It takes the + * O_NOFOLLOW branch, as after ENOSYS, EPERM or EINVAL. *openat2_used is + * set when the walk did not switch to that branch. + */ + +nxt_int_t +nxt_isolation_test_mount_dst_open(nxt_task_t *task, int rootfs_fd, + const char *rel, nxt_bool_t fallback, nxt_bool_t *openat2_used, int *fdp) +{ + nxt_int_t ret; + +#if (NXT_HAVE_OPENAT2) + nxt_isolation_no_openat2 = fallback; +#endif + + ret = nxt_isolation_mount_dst_open(task, rootfs_fd, (const u_char *) rel, + (const u_char *) rel, fdp); + +#if (NXT_HAVE_OPENAT2) + *openat2_used = !nxt_isolation_no_openat2; +#else + *openat2_used = 0; +#endif + + return ret; +} + +#endif + + nxt_int_t nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) { - size_t i, n; + int rootfs_fd, mnt_dst_fd; + u_char *mounted; + size_t i, n, rootfs_len; nxt_int_t ret; struct stat st; nxt_array_t *mounts; - const u_char *dst; + const u_char *dst, *rootfs_path; nxt_fs_mount_t *mnt; nxt_process_automount_t *automount; -#if (NXT_HAVE_OPENAT2) - int rootfs_fd; - const u_char *rootfs_path; - size_t rootfs_len; -#endif automount = &process->isolation.automount; mounts = process->isolation.mounts; @@ -892,35 +1134,42 @@ nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) } #endif -#if (NXT_HAVE_OPENAT2) /* - * Mount destinations live under a (possibly user-owned) rootfs. An - * attacker with write access to that tree can race mkdir->mount(2) - * by swapping a path component for a symlink pointing outside the - * rootfs. Re-open the rootfs and resolve each destination with - * RESOLVE_BENEATH so a tampered destination that escapes the - * rootfs fails the open, before we hand a path to mount(2). - * RESOLVE_NO_SYMLINKS is intentionally NOT requested: legitimate - * rootfs setups commonly contain in-tree symlinks (e.g. - * /lib -> /usr/lib) that must still resolve. + * Mount destinations live under a rootfs that may be writable by the + * application user. nxt_isolation_mount_dst_open() creates them + * without following symlinks and returns an fd. On Linux the mount + * targets that fd, not the path. */ rootfs_path = process->isolation.rootfs; - rootfs_len = (rootfs_path != NULL) ? nxt_strlen(rootfs_path) : 0; - - rootfs_fd = -1; - if (rootfs_len > 0) { - rootfs_fd = open((const char *) rootfs_path, - O_PATH | O_DIRECTORY | O_CLOEXEC); - if (rootfs_fd == -1) { - nxt_alert(task, "open rootfs(%s) %E", rootfs_path, nxt_errno); - return NXT_ERROR; - } - } + rootfs_len = nxt_strlen(rootfs_path); + + /* + * The rootfs is operator config. It must exist: a missing rootfs is + * refused, not created as root. + */ +#if defined(O_PATH) + rootfs_fd = open((const char *) rootfs_path, + O_PATH | O_DIRECTORY | O_CLOEXEC); +#else + rootfs_fd = open((const char *) rootfs_path, + O_RDONLY | O_DIRECTORY | O_CLOEXEC); #endif + if (nxt_slow_path(rootfs_fd == -1)) { + nxt_alert(task, "open rootfs(%s) %E", rootfs_path, nxt_errno); + return NXT_ERROR; + } - for (i = 0; i < n; i++) { - int mnt_dst_fd = -1; + /* + * mounted[i] is set when this call has mounted mnt[i]. The undo path + * unmounts only those. One extra byte, because n can be 0. + */ + mounted = nxt_zalloc(n + 1); + if (nxt_slow_path(mounted == NULL)) { + close(rootfs_fd); + return NXT_ERROR; + } + for (i = 0; i < n; i++) { dst = mnt[i].dst; if (mnt[i].deps && !automount->language_deps) { @@ -934,117 +1183,58 @@ nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) continue; } - ret = nxt_fs_mkdir_p(dst, 0777); - if (nxt_slow_path(ret != NXT_OK)) { - nxt_alert(task, "mkdir(%s) %E", dst, nxt_errno); - goto undo; - } - -#if (NXT_HAVE_OPENAT2) /* - * Require a path separator right after the rootfs prefix so a - * sibling like "-helper" is not treated as living under - * "" (which would resolve a bogus relative path and abort - * startup with a spurious ENOENT). dst is always built as - * rootfs + a component starting with '/', so this byte is '/' for - * every real destination; the preceding length check makes the - * index safe to read. + * nxt_isolation_set_lang_mounts() builds every destination as + * rootfs + module path. A module path from the build (for example + * sys.path) can be relative, so check that it starts with '/'. */ - if (rootfs_fd != -1 - && nxt_strlen(dst) > rootfs_len - && memcmp(dst, rootfs_path, rootfs_len) == 0 - && dst[rootfs_len] == '/') - { - struct open_how how; - const char *rel; - int fd; - - rel = (const char *) dst + rootfs_len; - while (*rel == '/') { - rel++; - } + nxt_assert(memcmp(dst, rootfs_path, rootfs_len) == 0); - nxt_memzero(&how, sizeof(how)); - how.flags = O_PATH | O_CLOEXEC | O_DIRECTORY; - how.resolve = RESOLVE_BENEATH; - - fd = syscall(SYS_openat2, rootfs_fd, rel, &how, sizeof(how)); - if (fd == -1) { - if (nxt_errno == ENOSYS) { - /* - * openat2(2) is Linux 5.6+; older kernels return - * ENOSYS even when the userspace headers are - * present. Skip the symlink-resolution check on - * such kernels rather than failing every isolation - * setup; mount(2) below still operates on the - * caller-supplied path, just without the extra - * RESOLVE_BENEATH guard. Warn once so operators - * see the reduced security posture. - */ - static nxt_bool_t openat2_warned; - if (!openat2_warned) { - nxt_log(task, NXT_LOG_WARN, - "openat2(SYS_openat2) not supported by the " - "running kernel; rootfs mount destinations " - "are not validated against symlinks"); - openat2_warned = 1; - } - - } else { - nxt_alert(task, "mount destination %s escapes rootfs %s " - "or contains symlinks: %E", dst, rootfs_path, - nxt_errno); - ret = NXT_ERROR; - goto undo; - } + if (nxt_slow_path(dst[rootfs_len] != '/')) { + nxt_alert(task, "mount destination %s is not an absolute path " + "under rootfs %s", dst, rootfs_path); + goto undo; + } - } else { - /* - * Keep the validated fd open and mount onto it directly - * (via /proc/self/fd/ in nxt_fs_mount) so the mount - * targets the exact inode openat2 resolved, rather than - * re-resolving dst as a path and reopening the check-then- - * use window. Closed after the mount below. - */ - mnt_dst_fd = fd; - } + ret = nxt_isolation_mount_dst_open(task, rootfs_fd, dst, + dst + rootfs_len + 1, &mnt_dst_fd); + if (nxt_slow_path(ret != NXT_OK)) { + goto undo; } -#endif ret = nxt_fs_mount(task, &mnt[i], mnt_dst_fd); -#if (NXT_HAVE_OPENAT2) - if (mnt_dst_fd != -1) { - close(mnt_dst_fd); - } -#endif + close(mnt_dst_fd); if (nxt_slow_path(ret != NXT_OK)) { goto undo; } - } -#if (NXT_HAVE_OPENAT2) - if (rootfs_fd != -1) { - close(rootfs_fd); + mounted[i] = 1; } -#endif + + nxt_free(mounted); + close(rootfs_fd); return NXT_OK; undo: - n = i + 1; + /* + * Unmount only the destinations that this call mounted. A skipped or + * failed destination is not mounted, and its path can go through a + * symlink. + */ + n = i; for (i = 0; i < n; i++) { - nxt_fs_unmount(mnt[i].dst); + if (mounted[i]) { + nxt_fs_unmount(mnt[i].dst); + } } -#if (NXT_HAVE_OPENAT2) - if (rootfs_fd != -1) { - close(rootfs_fd); - } -#endif + nxt_free(mounted); + close(rootfs_fd); return NXT_ERROR; } diff --git a/src/nxt_isolation.h b/src/nxt_isolation.h index b1bfc33c7..172db3317 100644 --- a/src/nxt_isolation.h +++ b/src/nxt_isolation.h @@ -13,6 +13,12 @@ nxt_int_t nxt_isolation_main_prefork(nxt_task_t *task, nxt_process_t *process, nxt_int_t nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process); nxt_int_t nxt_isolation_change_root(nxt_task_t *task, nxt_process_t *process); + +#if (NXT_TESTS) +/* The mount destination walk, for the mount destination test. */ +nxt_int_t nxt_isolation_test_mount_dst_open(nxt_task_t *task, int rootfs_fd, + const char *rel, nxt_bool_t fallback, nxt_bool_t *openat2_used, int *fdp); +#endif #endif #endif /* _NXT_ISOLATION_H_INCLUDED_ */ diff --git a/src/test/nxt_isolation_mount_dst_test.c b/src/test/nxt_isolation_mount_dst_test.c new file mode 100644 index 000000000..e8411d427 --- /dev/null +++ b/src/test/nxt_isolation_mount_dst_test.c @@ -0,0 +1,378 @@ +/* + * Copyright (C) FreeUnit + */ + +/* + * The mount destination walk in src/nxt_isolation.c has two branches. With + * openat2() it resolves each prefix beneath the rootfs. Without openat2() + * it opens each component with O_NOFOLLOW and refuses a symlink. Every CI + * kernel has openat2(), so the Python isolation tests run only the first + * branch. This test runs each case twice: once as unitd runs it, and once + * with the second branch forced through nxt_isolation_test_mount_dst_open(). + * + * The walk needs no privileges, so the test runs as any user. + * + * The rootfs of each case: + * + * decoy/ outside the rootfs, must stay empty + * r/usr -> ../decoy a link out of the rootfs + * r/a/esc -> ../../decoy the same link one level down + * r/real/lib/ + * r/lib -> real/lib a link inside the rootfs + */ + +#include +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + + +typedef struct { + const char *name; + const char *rel; + /* The directory that the walk opens, relative to the rootfs. */ + const char *opened; + /* The component that the walk refuses. */ + const char *comp; + /* Only openat2() resolves the destination. */ + nxt_bool_t openat2_only; +} nxt_isolation_mount_dst_case_t; + + +typedef struct { + const nxt_isolation_mount_dst_case_t *tc; + nxt_bool_t fallback; + nxt_bool_t have_openat2; + nxt_log_t *log; + const char *tmpdir; + u_char msg[NXT_MAX_ERROR_STR]; +} nxt_isolation_mount_dst_ctx_t; + + +static const nxt_isolation_mount_dst_case_t nxt_mount_dst_cases[] = { + { "symlink out of rootfs", "usr/lib/python3.12", NULL, "usr", 0 }, + { "symlink out of rootfs below a directory", "a/esc/lib", NULL, "esc", + 0 }, + { "symlink inside rootfs", "lib/x", "real/lib/x", "lib", 1 }, + { "no symlink", "a/b/c", "a/b/c", NULL, 0 }, +}; + + +static void nxt_cdecl +nxt_isolation_mount_dst_test_log_handler(nxt_uint_t level, nxt_log_t *log, + const char *fmt, ...) +{ + u_char *p, *end; + va_list args; + nxt_isolation_mount_dst_ctx_t *ctx; + + if (level != NXT_LOG_ALERT) { + return; + } + + ctx = log->ctx; + + p = ctx->msg; + end = ctx->msg + sizeof(ctx->msg) - 1; + + va_start(args, fmt); + p = nxt_vsprintf(p, end, fmt, args); + va_end(args); + + *p = '\0'; +} + + +static int +nxt_isolation_mount_dst_test_remove(const char *path, const struct stat *st, + int flag, struct FTW *ftw) +{ + (void) remove(path); + + return 0; +} + + +static nxt_bool_t +nxt_isolation_mount_dst_test_empty(int dir_fd, const char *name) +{ + int fd; + DIR *dir; + nxt_bool_t empty; + struct dirent *de; + + fd = openat(dir_fd, name, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (fd == -1) { + return 0; + } + + dir = fdopendir(fd); + if (dir == NULL) { + close(fd); + return 0; + } + + empty = 1; + + while ((de = readdir(dir)) != NULL) { + if (nxt_strcmp(de->d_name, ".") != 0 + && nxt_strcmp(de->d_name, "..") != 0) + { + empty = 0; + } + } + + closedir(dir); + + return empty; +} + + +static nxt_int_t +nxt_isolation_mount_dst_test_setup(int tmp_fd) +{ + if (mkdirat(tmp_fd, "decoy", 0777) != 0 + || mkdirat(tmp_fd, "r", 0777) != 0 + || mkdirat(tmp_fd, "r/a", 0777) != 0 + || mkdirat(tmp_fd, "r/real", 0777) != 0 + || mkdirat(tmp_fd, "r/real/lib", 0777) != 0 + || symlinkat("../decoy", tmp_fd, "r/usr") != 0 + || symlinkat("../../decoy", tmp_fd, "r/a/esc") != 0 + || symlinkat("real/lib", tmp_fd, "r/lib") != 0) + { + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_isolation_mount_dst_test_run(nxt_isolation_mount_dst_ctx_t *ctx, + int tmp_fd) +{ + int fd, rootfs_fd; + nxt_int_t ret; + nxt_log_t log; + nxt_bool_t openat2_used, use_openat2, refused; + nxt_task_t task; + struct stat st, want; + const char *mode; + const nxt_isolation_mount_dst_case_t *tc; + + tc = ctx->tc; + mode = ctx->fallback ? "fallback" : "default"; + + NXT_TEST_CHECK(ctx->log, + nxt_isolation_mount_dst_test_setup(tmp_fd) == NXT_OK, + "mount dst test: setup in \"%s\" failed %E", ctx->tmpdir, + nxt_errno); + + rootfs_fd = openat(tmp_fd, "r", O_RDONLY | O_DIRECTORY | O_CLOEXEC); + + NXT_TEST_CHECK(ctx->log, rootfs_fd != -1, + "mount dst test: open rootfs failed %E", nxt_errno); + + log.level = NXT_LOG_INFO; + log.ident = 0; + log.handler = nxt_isolation_mount_dst_test_log_handler; + log.ctx_handler = NULL; + log.ctx = ctx; + + nxt_memzero(&task, sizeof(task)); + task.thread = nxt_thread(); + task.log = &log; + + fd = -1; + + ret = nxt_isolation_test_mount_dst_open(&task, rootfs_fd, tc->rel, + ctx->fallback, &openat2_used, &fd); + + use_openat2 = !ctx->fallback && ctx->have_openat2; + refused = (tc->comp != NULL && (!tc->openat2_only || !use_openat2)); + + NXT_TEST_CHECK(ctx->log, nxt_isolation_mount_dst_test_empty(tmp_fd, + "decoy"), + "mount dst test: %s, %s: a directory was created " + "outside the rootfs", tc->name, mode); + + if (refused) { + NXT_TEST_CHECK(ctx->log, ret == NXT_ERROR, + "mount dst test: %s, %s: \"%s\" is not refused", + tc->name, mode, tc->rel); + + nxt_log_error(NXT_LOG_NOTICE, ctx->log, "mount dst test: %s, %s: %s", + tc->name, mode, ctx->msg); + + /* + * Each branch has its own message. Only Linux is checked: for a + * symlink opened with O_NOFOLLOW, FreeBSD returns EMLINK, and the + * fallback logs the errno instead. + */ +#if (NXT_LINUX) + { + u_char want_msg[128]; + + (void) nxt_sprintf(want_msg, want_msg + sizeof(want_msg), + "component \"%s\" %s%Z", tc->comp, + use_openat2 + ? "escapes rootfs or is not a directory: " + : "is a symlink or not a directory"); + + NXT_TEST_CHECK(ctx->log, + strstr((char *) ctx->msg, (char *) want_msg) + != NULL, + "mount dst test: %s, %s: the log is \"%s\", " + "wanted \"%s\"", tc->name, mode, ctx->msg, + want_msg); + } +#endif + + if (tc->opened != NULL) { + NXT_TEST_CHECK(ctx->log, + fstatat(rootfs_fd, tc->opened, &st, 0) != 0, + "mount dst test: %s, %s: \"%s\" was created", + tc->name, mode, tc->opened); + } + + } else { + NXT_TEST_CHECK(ctx->log, ret == NXT_OK, + "mount dst test: %s, %s: \"%s\" is refused: %s", + tc->name, mode, tc->rel, ctx->msg); + + NXT_TEST_CHECK(ctx->log, + fstat(fd, &st) == 0 + && fstatat(rootfs_fd, tc->opened, &want, 0) == 0 + && st.st_dev == want.st_dev + && st.st_ino == want.st_ino, + "mount dst test: %s, %s: the walk did not open " + "\"%s\"", tc->name, mode, tc->opened); + + close(fd); + } + + NXT_TEST_CHECK(ctx->log, openat2_used == use_openat2, + "mount dst test: %s, %s: openat2 used %d, wanted %d", + tc->name, mode, (int) openat2_used, (int) use_openat2); + + close(rootfs_fd); + + return NXT_OK; +} + + +static int +nxt_isolation_mount_dst_test_child(void *data) +{ + int tmp_fd; + char dir[256]; + nxt_int_t ret; + nxt_isolation_mount_dst_ctx_t *ctx; + + ctx = data; + + (void) nxt_sprintf((u_char *) dir, (u_char *) dir + sizeof(dir), + "%s/nxt_isolation_mount_dst_test.XXXXXX%Z", + ctx->tmpdir); + + if (mkdtemp(dir) == NULL) { + nxt_log_alert(ctx->log, "mount dst test: mkdtemp(\"%s\") failed %E", + dir, nxt_errno); + return 1; + } + + tmp_fd = open(dir, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + + if (tmp_fd == -1) { + nxt_log_alert(ctx->log, "mount dst test: open(\"%s\") failed %E", + dir, nxt_errno); + ret = NXT_ERROR; + + } else { + ret = nxt_isolation_mount_dst_test_run(ctx, tmp_fd); + close(tmp_fd); + } + + (void) nftw(dir, nxt_isolation_mount_dst_test_remove, 16, + FTW_DEPTH | FTW_PHYS); + + return (ret == NXT_OK) ? 0 : 1; +} + + +#if (NXT_HAVE_OPENAT2) + +static nxt_bool_t +nxt_isolation_mount_dst_test_have_openat2(void) +{ + int fd; + struct open_how how; + + nxt_memzero(&how, sizeof(how)); + how.flags = O_PATH | O_DIRECTORY | O_CLOEXEC; + how.resolve = RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS; + + fd = syscall(SYS_openat2, AT_FDCWD, ".", &how, sizeof(how)); + if (fd == -1) { + return 0; + } + + close(fd); + + return 1; +} + +#endif + + +nxt_int_t +nxt_isolation_mount_dst_test(nxt_thread_t *thr) +{ + nxt_uint_t i, fallback; + nxt_isolation_mount_dst_ctx_t ctx; + + nxt_thread_time_update(thr); + + nxt_memzero(&ctx, sizeof(ctx)); + + ctx.log = thr->log; + + ctx.tmpdir = getenv("TMPDIR"); + if (ctx.tmpdir == NULL || ctx.tmpdir[0] != '/') { + ctx.tmpdir = "/tmp"; + } + +#if (NXT_HAVE_OPENAT2) + ctx.have_openat2 = nxt_isolation_mount_dst_test_have_openat2(); +#endif + + if (!ctx.have_openat2) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "mount dst test: openat2() " + "is not available, both runs take the fallback"); + } + + for (fallback = 0; fallback <= 1; fallback++) { + for (i = 0; i < nxt_nitems(nxt_mount_dst_cases); i++) { + ctx.tc = &nxt_mount_dst_cases[i]; + ctx.fallback = fallback; + + if (nxt_test_in_child(thr, ctx.tc->name, + nxt_isolation_mount_dst_test_child, &ctx) + != 0) + { + nxt_log_alert(thr->log, "mount dst test: \"%s\" failed " + "with the %s branch", ctx.tc->name, + fallback ? "fallback" : "default"); + return NXT_ERROR; + } + } + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "mount destination test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 5d29ce22b..78e4e0f9a 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -429,5 +429,11 @@ main(int argc, char **argv) } #endif +#if (NXT_HAVE_ISOLATION_ROOTFS) + if (nxt_isolation_mount_dst_test(thr) != NXT_OK) { + return 1; + } +#endif + return 0; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 31a2c90f7..ca240ef83 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -121,6 +121,9 @@ nxt_int_t nxt_openssl_server_init_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_controller_peer_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); +#if (NXT_HAVE_ISOLATION_ROOTFS) +nxt_int_t nxt_isolation_mount_dst_test(nxt_thread_t *thr); +#endif nxt_bool_t nxt_test_fd_is_open(nxt_fd_t fd); int nxt_test_in_child(nxt_thread_t *thr, const char *name, int (*fn)(void *), diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index 87164cb36..f48654fcd 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -127,6 +127,172 @@ def test_python_isolation_rootfs_no_language_deps(require, temp_dir): ), 'language_deps unmount' +# The first component of a language_deps destination: "usr" for a distro +# Python, "opt" for a toolcache Python (as in CI). +LANG_DEPS_TOP = ('usr', 'opt') + + +def lang_deps_tops(temp_dir): + # Return the members of LANG_DEPS_TOP that start a language_deps + # destination. The destinations come from the module build. To find + # them, load the application once with a rootfs that has no symlink, + # and read the mount table. + rootfs = Path(temp_dir) / 'plain' + rootfs.mkdir() + + client.load('empty', isolation={'rootfs': str(rootfs)}) + + assert client.get()['status'] == 200, 'plain rootfs' + assert waitfor( + lambda: bool(language_deps_mounts(str(rootfs))) + ), 'plain rootfs language_deps mount' + + tops = { + Path(line.split(' ')[0]).relative_to(rootfs).parts[0] + for line in language_deps_mounts(str(rootfs)) + } + + client.conf({"listeners": {}, "applications": {}}) + + assert waitfor( + lambda: not language_deps_mounts(str(rootfs)) + ), 'plain rootfs language_deps unmount' + + return sorted(tops.intersection(LANG_DEPS_TOP)) + + +def symlinked_rootfs(temp_dir): + # Make each LANG_DEPS_TOP in the rootfs a symlink out of the rootfs. + # mkdir -p follows the symlink and creates decoy/lib/... or + # decoy/hostedtoolcache/... + rootfs = Path(temp_dir) / 'r' + decoy = Path(temp_dir) / 'decoy' + + rootfs.mkdir() + decoy.mkdir() + + for top in LANG_DEPS_TOP: + (rootfs / top).symlink_to('../decoy') + + return rootfs, decoy + + +def wait_for_refusal(rootfs, tops, findall, wait_for_record): + # Wait for the record of the refused destination. Return the pid of + # the prototype and the destination. + tops_re = '|'.join(re.escape(top) for top in tops) + record = ( + fr'\[alert\] (\d+)#\d+ mount destination ' + fr'({re.escape(str(rootfs))}/(?:{tops_re})/\S+): ' + fr'component "(?:{tops_re})" (.*)$' + ) + + found = wait_for_record(record, wait=50) + assert found is not None, 'refusal logged' + + # The walk logs one reason with openat2() and another without it. + # unitd uses openat2() when the build has it, unless the kernel refuses + # the call. Then unitd logs a warning and takes the fallback. + config = Path(option.current_dir) / 'build/include/nxt_auto_config.h' + have_openat2 = re.search( + r'^#define NXT_HAVE_OPENAT2\b', config.read_text(), re.M + ) + no_openat2 = findall(r'openat2\(\) is not supported by the running kernel') + + if have_openat2 and not no_openat2: + reason = 'escapes rootfs or is not a directory: ' + else: + reason = 'is a symlink or not a directory' + + assert found.group(3).startswith(reason), 'refusal reason' + + return found.group(1), found.group(2) + + +def test_python_isolation_rootfs_symlinked_mount_dst( + findall, require, skip_alert, temp_dir, wait_for_record +): + require({'privileged_user': True}) + + # A destination whose path goes through a symlink out of the rootfs + # is refused, and no directory is created behind the symlink. + tops = lang_deps_tops(temp_dir) + if not tops: + pytest.skip('no language_deps destination under /usr or /opt') + + rootfs, decoy = symlinked_rootfs(temp_dir) + + skip_alert(r'mount destination .*', r'failed to apply', r'process .* exited') + + client.load('empty', isolation={'rootfs': str(rootfs)}) + + assert client.get()['status'] != 200, 'symlinked destination refused' + + wait_for_refusal(rootfs, tops, findall, wait_for_record) + + assert sorted(p.name for p in decoy.iterdir()) == [], 'decoy is empty' + + +def test_python_isolation_rootfs_symlinked_mount_dst_no_unmount( + findall, require, skip_alert, temp_dir, wait_for_record +): + require({'privileged_user': True}) + + # When the prototype refuses a destination because of a symlink, it + # must not unmount that destination. The path goes through the + # symlink, and without a mount namespace umount2() runs in the mount + # namespace of unitd. + tops = lang_deps_tops(temp_dir) + if not tops: + pytest.skip('no language_deps destination under /usr or /opt') + + rootfs, _ = symlinked_rootfs(temp_dir) + + skip_alert(r'mount destination .*', r'failed to apply', r'process .* exited') + + client.load('empty', isolation={'rootfs': str(rootfs)}) + client.get() + + pid, dst = wait_for_refusal(rootfs, tops, findall, wait_for_record) + dst = re.escape(dst) + + assert wait_for_record(fr'process {pid} exited'), 'prototype exited' + + assert not findall( + fr'\] {pid}#\d+ umount2\({dst},' + ), 'refused destination is not unmounted' + + +def test_python_isolation_rootfs_missing(require, skip_alert, temp_dir): + require({'privileged_user': True}) + + # A missing rootfs is refused. It is not created as root. The conf + # is sent directly: client.load() would create the rootfs. + missing = Path(temp_dir) / 'missing' + script_path = f'{option.test_dir}/python/empty' + + skip_alert(r'open rootfs', r'failed to apply', r'process .* exited') + + assert 'success' in client.conf( + { + "listeners": {"*:8080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "python", + "processes": {"spare": 0}, + "path": script_path, + "working_directory": script_path, + "module": "wsgi", + "isolation": {"rootfs": str(missing / 'r')}, + } + }, + } + ) + + assert client.get()['status'] != 200, 'missing rootfs refused' + assert not missing.exists(), 'missing rootfs not created' + + def test_python_isolation_rootfs_credential_language_deps( is_su, require, temp_dir ): From 25ec9f6e4fe12b34a3aafb7b8ea9e61cc9e26936 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 05:41:41 +0200 Subject: [PATCH 347/391] fix(isolation): unshare the cgroup namespace after the cgroup move isolation.namespaces.cgroup with isolation.cgroup.path rooted the app's cgroup namespace at the main process cgroup. The app's own /proc/self/cgroup showed 0::/scope/python instead of 0::/. nxt_process_create() forks the prototype. The child's first action was unshare() with the full clone flag set, CLONE_NEWCGROUP included. Per cgroup_namespaces(7) a new cgroup namespace is rooted at the cgroup of the calling process at unshare() time. The parent wrote the pid into the target cgroup later, in nxt_cgroup_proc_add(), so the namespace root was an ancestor of the app cgroup. With isolation.namespaces.pid the order was deterministic. The parent blocked in nxt_process_recv_pid() until after the child unshared and double forked. Without pid isolation no pipe was created at all, so the child raced the parent and almost always won. The parent now moves the child into its cgroup right after fork(), by the pid that fork() returned. nxt_cgroup_proc_add() takes that pid as an argument. The child waits for a status byte from the parent before its unshare() call and before the pid isolation double fork. A forked process starts in the cgroup of its parent, so the grandchild is in the cgroup too. The child keeps one unshare() call for all flags. With CLONE_NEWUSER, the kernel checks the other flags with the credentials from before the call. A second unshare(CLONE_NEWCGROUP) after CLONE_NEWUSER uses the new credentials. Ubuntu 24.04 sets kernel.apparmor_restrict_unprivileged_userns to 1. There an unprivileged process that creates a user namespace moves to the unprivileged_userns AppArmor profile, and that profile denies capabilities. A second call gets EPERM, so a non-root unitd with credential and cgroup namespaces could not start the application. Each case with a cgroup path now waits for the move, also without a cgroup namespace. With pid isolation the move must come before the double fork, because the grandchild would stay in the old cgroup. Without namespaces the prototype no longer runs in the main process cgroup before the move. nxt_process_init_pipes() replaces nxt_process_init_pidns(). It creates gc_pipe when isolation.cgroup.path is set or with pid isolation, and pid_pipe only with pid isolation. It sets both pipes to -1 first. gc_pipe carries one status byte for each wait. With a cgroup path the parent sends 0 after nxt_cgroup_proc_add(). With pid isolation the parent sends 0 after it has read the pid of the grandchild, as before. -1, end of file or the 5 second timeout of nxt_process_pipe_timer() make the waiting process exit. nxt_process_check_pid_status() is now nxt_process_recv_status(), because it also reads the cgroup status byte. It no longer closes the pipe. The status write moved out of nxt_process_recv_pid() into the new nxt_process_send_status(). The parent keeps its read end of gc_pipe open until it has sent the last byte, so a write never gets EPIPE. When nxt_cgroup_proc_add() fails, the parent sends -1 and returns. The child exits before unshare(). kill(pid, SIGTERM) stays only for builds without NXT_HAVE_LINUX_NS, where the child does not wait. Before, the child could already run when the parent killed it. The pipes are now closed when the pid pipe cannot be created and when fork() fails. When nxt_process_recv_pid() fails, the parent calls nxt_cgroup_cleanup(). If the child is still in the cgroup, rmdir() fails and the directory stays. test_python_isolation_cgroup_namespace() loads ns_inspect with isolation {namespaces: {cgroup: true}}, one time with cgroup {path: 'scope/python'} and one time without a cgroup path. It asserts that the app's own /proc/self/cgroup is exactly 0::/. Before the fix the case with the path shows 0::/scope/python. With the path it also asserts that /proc//cgroup of the application, read by the test, ends in scope/python. The view inside is 0::/ also when the process is not moved. test/python/ns_inspect/wsgi.py gained a read= parameter that returns the file content as a JSON string. The existing path= parameter is unchanged. The comment in nxt_cgroup_proc_add() no longer says that CLONE_NEWCGROUP is a reason to read /proc//cgroup. The child has not unshared the cgroup namespace at that time. The application was always placed in the configured cgroup. The parent writes the pid by absolute path, so the limits applied. Only the namespace root was wrong. CHANGES and docs/changes.xml have a 1.36.2 bugfix entry. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 6 + docs/changes.xml | 10 ++ src/nxt_cgroup.c | 17 +- src/nxt_cgroup.h | 3 +- src/nxt_process.c | 275 +++++++++++++++++++++++---------- test/python/ns_inspect/wsgi.py | 11 +- test/test_python_isolation.py | 29 ++++ 7 files changed, 258 insertions(+), 93 deletions(-) diff --git a/CHANGES b/CHANGES index fcf203145..1462877fc 100644 --- a/CHANGES +++ b/CHANGES @@ -84,6 +84,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 directory is now refused on every system. Builds without openat2(), such as on FreeBSD, created it before. + *) Bugfix: with isolation.namespaces.cgroup and isolation.cgroup.path, + the cgroup namespace of an application was rooted at the cgroup of + the main process, not at the configured cgroup. The application saw + its own cgroup as "/scope/python" instead of "/". The namespace is + now created after the process is moved into its cgroup. + *) Security: the router and libunit read records from shared memory that the other process can still write, without enough checks. A buffer that was not a whole number of records was read past its diff --git a/docs/changes.xml b/docs/changes.xml index a0f3e3f1d..863f55cdf 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -126,6 +126,16 @@ created it before. + + +with isolation.namespaces.cgroup and isolation.cgroup.path, the cgroup +namespace of an application was rooted at the cgroup of the main process, not +at the configured cgroup. The application saw its own cgroup as +"/scope/python" instead of "/". The namespace is now created after the process +is moved into its cgroup. + + + when unitd ran as root and the control socket was given to another user or diff --git a/src/nxt_cgroup.c b/src/nxt_cgroup.c index fd9d72449..86896599a 100644 --- a/src/nxt_cgroup.c +++ b/src/nxt_cgroup.c @@ -87,7 +87,7 @@ nxt_cgroup_rmdir_up(char *cgpath, const char *cgroot) nxt_int_t -nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process) +nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process, nxt_pid_t pid) { int len; size_t old_len; @@ -103,15 +103,12 @@ nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process) } /* - * Resolve the cgroup path against /proc//cgroup rather than - * /proc/self/cgroup: the parent's cgroup view may differ from the - * just-forked child's, particularly when CLONE_NEWCGROUP is in play - * and the configured path is relative. Reading the child's own - * /proc entry avoids a TOCTOU where the parent moves between cgroups - * after fork() but before this write. + * Resolve the cgroup path from /proc//cgroup, not from + * /proc/self/cgroup. The parent can move to a different cgroup after + * fork() and before this write. A relative path that uses the + * parent's entry then points to the wrong cgroup. */ - ret = nxt_mk_cgpath(task, process->isolation.cgroup.path, cgprocs, - process->pid); + ret = nxt_mk_cgpath(task, process->isolation.cgroup.path, cgprocs, pid); if (nxt_slow_path(ret == NXT_ERROR)) { return NXT_ERROR; } @@ -168,7 +165,7 @@ nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process) } setvbuf(fp, NULL, _IONBF, 0); - len = fprintf(fp, "%d\n", process->pid); + len = fprintf(fp, "%d\n", pid); nxt_file_fclose(task, fp); if (nxt_slow_path(len < 0)) { diff --git a/src/nxt_cgroup.h b/src/nxt_cgroup.h index 9d8c98a47..3ff26a78b 100644 --- a/src/nxt_cgroup.h +++ b/src/nxt_cgroup.h @@ -8,7 +8,8 @@ nxt_int_t nxt_cgroup_make_procs_path(char *cgprocs, size_t len); -nxt_int_t nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process); +nxt_int_t nxt_cgroup_proc_add(nxt_task_t *task, nxt_process_t *process, + nxt_pid_t pid); void nxt_cgroup_cleanup(nxt_task_t *task, const nxt_process_t *process); diff --git a/src/nxt_process.c b/src/nxt_process.c index 08426ba44..4dba7fd64 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -31,15 +31,18 @@ #if (NXT_HAVE_LINUX_NS) static nxt_int_t nxt_process_pipe_timer(nxt_fd_t fd, short event); -static nxt_int_t nxt_process_check_pid_status(const nxt_fd_t *gc_pipe); -static nxt_pid_t nxt_process_recv_pid(const nxt_fd_t *pid_pipe, - const nxt_fd_t *gc_pipe); +static void nxt_process_fd_close(nxt_fd_t *fd); +static void nxt_process_pipe_close(nxt_fd_t *pp); +static nxt_int_t nxt_process_recv_status(const nxt_fd_t *gc_pipe); +static void nxt_process_send_status(const nxt_fd_t *gc_pipe, int8_t status); +static nxt_pid_t nxt_process_recv_pid(const nxt_fd_t *pid_pipe); static void nxt_process_send_pid(const nxt_fd_t *pid_pipe, nxt_pid_t pid); static nxt_int_t nxt_process_unshare(nxt_task_t *task, nxt_process_t *process, - nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, nxt_bool_t use_pidns); -static nxt_int_t nxt_process_init_pidns(nxt_task_t *task, + nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, nxt_bool_t use_pidns, + nxt_bool_t use_cgroup); +static nxt_int_t nxt_process_init_pipes(nxt_task_t *task, const nxt_process_t *process, nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, - nxt_bool_t *use_pidns); + nxt_bool_t *use_pidns, nxt_bool_t *use_cgroup); #endif static nxt_pid_t nxt_process_create(nxt_task_t *task, nxt_process_t *process); @@ -445,51 +448,74 @@ nxt_process_pipe_timer(nxt_fd_t fd, short event) } +static void +nxt_process_fd_close(nxt_fd_t *fd) +{ + if (*fd != -1) { + close(*fd); + *fd = -1; + } +} + + +static void +nxt_process_pipe_close(nxt_fd_t *pp) +{ + /* A pipe that nxt_process_init_pipes() did not create is -1. */ + + nxt_process_fd_close(&pp[0]); + nxt_process_fd_close(&pp[1]); +} + + static nxt_int_t -nxt_process_check_pid_status(const nxt_fd_t *gc_pipe) +nxt_process_recv_status(const nxt_fd_t *gc_pipe) { int8_t status = -1; ssize_t ret; - close(gc_pipe[1]); - ret = nxt_process_pipe_timer(gc_pipe[0], POLLIN); if (ret == NXT_OK) { read(gc_pipe[0], &status, sizeof(int8_t)); } - close(gc_pipe[0]); - return status; } +static void +nxt_process_send_status(const nxt_fd_t *gc_pipe, int8_t status) +{ + /* + * The parent keeps its read end open until nxt_process_pipe_close(). + * So this write does not get EPIPE when the child has exited. + */ + if (gc_pipe[1] != -1) { + write(gc_pipe[1], &status, sizeof(int8_t)); + } +} + + static nxt_pid_t -nxt_process_recv_pid(const nxt_fd_t *pid_pipe, const nxt_fd_t *gc_pipe) +nxt_process_recv_pid(const nxt_fd_t *pid_pipe) { - int8_t status; ssize_t ret; nxt_pid_t pid; close(pid_pipe[1]); - close(gc_pipe[0]); - status = 0; + pid = -1; ret = nxt_process_pipe_timer(pid_pipe[0], POLLIN); if (ret == NXT_OK) { ret = read(pid_pipe[0], &pid, sizeof(nxt_pid_t)); - } - if (ret <= 0) { - status = -1; - pid = -1; + if (ret <= 0) { + pid = -1; + } } - write(gc_pipe[1], &status, sizeof(int8_t)); - close(pid_pipe[0]); - close(gc_pipe[1]); return pid; } @@ -514,86 +540,132 @@ nxt_process_send_pid(const nxt_fd_t *pid_pipe, nxt_pid_t pid) static nxt_int_t nxt_process_unshare(nxt_task_t *task, nxt_process_t *process, nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, - nxt_bool_t use_pidns) + nxt_bool_t use_pidns, nxt_bool_t use_cgroup) { int ret; nxt_pid_t pid; - if (process->isolation.clone.flags == 0) { - return NXT_OK; - } - - ret = unshare(process->isolation.clone.flags); - if (nxt_slow_path(ret == -1)) { - nxt_alert(task, "unshare() failed for %s %E", process->name, - nxt_errno); + /* + * The child only reads gc_pipe. Without its own write end, a read + * gets end of file when the parent exits. + */ + nxt_process_fd_close(&gc_pipe[1]); - if (use_pidns) { - nxt_pipe_close(task, gc_pipe); - nxt_pipe_close(task, pid_pipe); + if (use_cgroup) { + /* + * Wait until the parent has moved this process into its cgroup. + * A new cgroup namespace is rooted at the cgroup of the process + * at unshare() time. A process from the fork(2) below starts in + * the cgroup of this process. + */ + if (nxt_process_recv_status(gc_pipe) != 0) { + goto fail; } - - return NXT_ERROR; - } - - if (!use_pidns) { - return NXT_OK; } /* - * PID namespace requested. Employ a double fork(2) technique - * so that the prototype process will be placed into the new - * namespace and end up with PID 1 (as before with clone). + * Unshare all flags in one call. With CLONE_NEWUSER, the kernel checks + * the other flags with the credentials from before the call. A second + * unshare() after CLONE_NEWUSER uses the new credentials and can fail + * with EPERM, for example in the unprivileged_userns AppArmor profile + * of Ubuntu 24.04. */ - pid = fork(); - if (nxt_slow_path(pid < 0)) { - nxt_alert(task, "fork() failed for %s %E", process->name, nxt_errno); - nxt_pipe_close(task, gc_pipe); - nxt_pipe_close(task, pid_pipe); + if (process->isolation.clone.flags != 0) { + ret = unshare(process->isolation.clone.flags); + if (nxt_slow_path(ret == -1)) { + nxt_alert(task, "unshare() failed for %s %E", process->name, + nxt_errno); + goto fail; + } + } - return NXT_ERROR; + if (use_pidns) { + /* + * PID namespace requested. Employ a double fork(2) technique + * so that the prototype process will be placed into the new + * namespace and end up with PID 1 (as before with clone). + */ + pid = fork(); + if (nxt_slow_path(pid < 0)) { + nxt_alert(task, "fork() failed for %s %E", process->name, + nxt_errno); + goto fail; - } else if (pid > 0) { - nxt_pipe_close(task, gc_pipe); - nxt_process_send_pid(pid_pipe, pid); + } else if (pid > 0) { + nxt_process_pipe_close(gc_pipe); + nxt_process_send_pid(pid_pipe, pid); - _exit(EXIT_SUCCESS); - } + _exit(EXIT_SUCCESS); + } - nxt_pipe_close(task, pid_pipe); - ret = nxt_process_check_pid_status(gc_pipe); - if (ret == -1) { - return NXT_ERROR; + nxt_process_pipe_close(pid_pipe); + + /* Wait until the parent has the pid of this process. */ + + if (nxt_process_recv_status(gc_pipe) != 0) { + goto fail; + } } + nxt_process_pipe_close(gc_pipe); + return NXT_OK; + +fail: + + nxt_process_pipe_close(gc_pipe); + nxt_process_pipe_close(pid_pipe); + + return NXT_ERROR; } static nxt_int_t -nxt_process_init_pidns(nxt_task_t *task, const nxt_process_t *process, +nxt_process_init_pipes(nxt_task_t *task, const nxt_process_t *process, nxt_fd_t *pid_pipe, nxt_fd_t *gc_pipe, - nxt_bool_t *use_pidns) + nxt_bool_t *use_pidns, nxt_bool_t *use_cgroup) { - int ret; + int ret; + + pid_pipe[0] = pid_pipe[1] = -1; + gc_pipe[0] = gc_pipe[1] = -1; *use_pidns = 0; + *use_cgroup = 0; #if (NXT_HAVE_CLONE_NEWPID) *use_pidns = nxt_is_pid_isolated(process); #endif - if (!*use_pidns) { +#if (NXT_HAVE_CGROUP) + *use_cgroup = (process->isolation.cgroup.path != NULL); +#endif + + if (!*use_pidns && !*use_cgroup) { return NXT_OK; } - ret = nxt_pipe_create(task, pid_pipe, 0, 0); + /* + * gc_pipe carries status bytes from the parent to the new process. + * With a cgroup path, the parent sends 0 after it has moved the child + * into the cgroup. The child waits for it before unshare() and before + * the pid isolation fork(2). With pid isolation, the parent sends 0 + * after it has read the pid of the grandchild from pid_pipe. The + * grandchild waits for it. -1, end of file or a timeout make the + * process exit. + */ + ret = nxt_pipe_create(task, gc_pipe, 0, 0); if (nxt_slow_path(ret == NXT_ERROR)) { return NXT_ERROR; } - ret = nxt_pipe_create(task, gc_pipe, 0, 0); + if (!*use_pidns) { + return NXT_OK; + } + + ret = nxt_pipe_create(task, pid_pipe, 0, 0); if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_process_pipe_close(gc_pipe); return NXT_ERROR; } @@ -620,9 +692,10 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) #if (NXT_HAVE_LINUX_NS) nxt_fd_t pid_pipe[2], gc_pipe[2]; - nxt_bool_t use_pidns; + nxt_bool_t use_pidns, use_cgroup; - ret = nxt_process_init_pidns(task, process, pid_pipe, gc_pipe, &use_pidns); + ret = nxt_process_init_pipes(task, process, pid_pipe, gc_pipe, + &use_pidns, &use_cgroup); if (ret == NXT_ERROR) { return -1; } @@ -631,6 +704,12 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) pid = fork(); if (nxt_slow_path(pid < 0)) { nxt_alert(task, "fork() failed for %s %E", process->name, nxt_errno); + +#if (NXT_HAVE_LINUX_NS) + nxt_process_pipe_close(gc_pipe); + nxt_process_pipe_close(pid_pipe); +#endif + return pid; } @@ -638,7 +717,8 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) /* Child. */ #if (NXT_HAVE_LINUX_NS) - ret = nxt_process_unshare(task, process, pid_pipe, gc_pipe, use_pidns); + ret = nxt_process_unshare(task, process, pid_pipe, gc_pipe, + use_pidns, use_cgroup); if (ret == NXT_ERROR) { _exit(EXIT_FAILURE); } @@ -666,12 +746,56 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) nxt_debug(task, "fork(%s): %PI", process->name, pid); +#if (NXT_HAVE_CGROUP) + /* + * Move the child into its cgroup by the pid from fork(). The child + * waits for the status byte below before unshare() and before the pid + * isolation fork(2). So a new cgroup namespace is rooted at this + * cgroup, and the grandchild starts in it. + */ + ret = nxt_cgroup_proc_add(task, process, pid); + if (nxt_slow_path(ret != NXT_OK)) { + nxt_alert(task, "cgroup: failed to add process %s to %s %E", + process->name, process->isolation.cgroup.path, nxt_errno); + nxt_cgroup_cleanup(task, process); + #if (NXT_HAVE_LINUX_NS) + /* The child waits for the status byte. -1 makes it exit. */ + + nxt_process_send_status(gc_pipe, -1); + nxt_process_pipe_close(gc_pipe); + nxt_process_pipe_close(pid_pipe); +#else + kill(pid, SIGTERM); +#endif + + return -1; + } +#endif + +#if (NXT_HAVE_LINUX_NS) + if (use_cgroup) { + nxt_process_send_status(gc_pipe, 0); + } + if (use_pidns) { - pid = nxt_process_recv_pid(pid_pipe, gc_pipe); - if (pid == -1) { - return pid; + pid = nxt_process_recv_pid(pid_pipe); + nxt_process_send_status(gc_pipe, (pid == -1) ? -1 : 0); + } + + nxt_process_pipe_close(gc_pipe); + + if (pid == -1) { +#if (NXT_HAVE_CGROUP) + if (use_cgroup) { + /* + * The child can still be in the cgroup. Then rmdir() fails + * and the directory stays. + */ + nxt_cgroup_cleanup(task, process); } +#endif + return pid; } #endif @@ -693,17 +817,6 @@ nxt_process_create(nxt_task_t *task, nxt_process_t *process) nxt_runtime_process_add(task, process); } -#if (NXT_HAVE_CGROUP) - ret = nxt_cgroup_proc_add(task, process); - if (nxt_slow_path(ret != NXT_OK)) { - nxt_alert(task, "cgroup: failed to add process %s to %s %E", - process->name, process->isolation.cgroup.path, nxt_errno); - nxt_cgroup_cleanup(task, process); - kill(pid, SIGTERM); - return -1; - } -#endif - return pid; } diff --git a/test/python/ns_inspect/wsgi.py b/test/python/ns_inspect/wsgi.py index fa1222e4f..d7798bfa8 100644 --- a/test/python/ns_inspect/wsgi.py +++ b/test/python/ns_inspect/wsgi.py @@ -16,7 +16,16 @@ def application(environ, start_response): d = parse_qs(environ['QUERY_STRING']) - ret['FileExists'] = os.path.exists(d.get('path')[0]) + if 'path' in d: + ret['FileExists'] = os.path.exists(d['path'][0]) + + if 'read' in d: + try: + with open(d['read'][0], 'r') as f: + ret['FileContent'] = f.read() + + except IOError: + ret['FileContent'] = None out = json.dumps(ret) diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index f48654fcd..d0b222819 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -395,6 +395,35 @@ def set_cgroup_path(path): assert len(cgroup_rel.parts) >= len(cgroup_abs.parts) +@pytest.mark.parametrize( + 'cgroup', [{'path': 'scope/python'}, None], ids=['path', 'no_path'] +) +def test_python_isolation_cgroup_namespace(require, cgroup): + require({'privileged_user': True, 'features': {'isolation': ['cgroup']}}) + + isolation = {'namespaces': {'cgroup': True}} + + if cgroup is not None: + isolation['cgroup'] = cgroup + + client.load('ns_inspect', isolation=isolation) + + ret = client.getjson(url='/?read=/proc/self/cgroup') + + content = ret['body']['FileContent'] + assert content is not None, 'read /proc/self/cgroup' + assert [ + l for l in content.splitlines() if l.startswith('0::') + ] == ['0::/'], 'cgroup namespace is rooted at the app cgroup' + + if cgroup is not None: + # The view inside is "/" also when the process is not moved. + assert Path(get_cgroup('ns_inspect')).parts[-2:] == ( + 'scope', + 'python', + ), 'app process is in the configured cgroup' + + def test_python_isolation_cgroup_two(require): require({'privileged_user': True, 'features': {'isolation': ['cgroup']}}) From 39700f29466f4e979efcdc1827ec2baac688b24e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 3 Oct 2026 22:12:47 +0200 Subject: [PATCH 348/391] libunit: keep the messages of a request in order on a queue port A router engine port has a shared memory queue and a socket. libunit puts a message of up to 31 bytes into the queue. A larger message goes to the socket, and only a 1-byte READ_SOCKET marker goes into the queue. For each marker, the router reads the next datagram from the socket (nxt_port_queue_read_handler()). The marker does not say who sent it, and all processes of an application write to the same port. So a datagram can be read before a queue message that its sender sent earlier. Process Y puts its marker into the queue and is preempted before its sendmsg(). Process X then puts a response header of more than 1024 bytes into the queue, as a 28-byte shared memory message, and sends a body of 16 to 1024 bytes to the socket. The router reads the marker of Y first and takes the body of X for it. The request has no header yet, so the router parses the body as the header, logs "response buffer too small for fields count: 574235237", and answers 503. 574235237 is 0x223a2265: bytes 8 to 11 of a JSON body that starts with {"message":". The same race can put the parts of a body in the wrong order. A write over 1024 bytes goes into the queue, and a later smaller write can reach the client first. Nothing is logged then. The reverse cannot happen. A sender puts its marker into the queue before it sends the datagram, so the router reads a datagram before any queue message that its sender sent after it. So libunit now puts a message of a request into the queue only if it is the last message of the request. The acknowledgement of the request headers, the response header and the body parts go to the socket. The last message and the messages of no request (stream 0) still use the queue. The router writes to an application port with the same markers. This change does not touch that direction. The new case in unit_msg_test plays process Y and the router. Before this change, the router gets the messages of the request in the order "ABHL" (ack, body, header, last). After it, the order is "AHBL". With 64 router threads, up to 64 PHP processes and 32 clients, and a script that answers 404 with a JSON body as Drupal does, the router answered 662 of 300,000 requests with 503 on ac2f2d29, and 438 on b12fb677. With this change, it answered none. A response written as 2000 bytes and then 100 bytes reached the client in the wrong order 39 times in 96,000 requests before this change, and 0 times after it. The cost is a sendmsg() for messages that used the queue. With one PHP process and 4 clients, 2000 requests made 6,343 sendmsg() calls in the application before and 6,396 after. With a header over 1024 bytes, they made 5,830 before and 6,258 after. The bug had appeared in 1.19.0, with the shared memory queue. One limit stays. libunit puts the marker into the queue before it calls sendmsg(). If sendmsg() fails, the marker has no datagram. The router waits at it, and the queue messages behind it wait for the next datagram from any process. Each later marker takes the datagram meant for the one before it, so the last message of a quiet port waits for the next datagram from any process. This was already so for a message over 31 bytes. Now it also applies to the acknowledgement and to the other messages of a request before the last one. On Linux 7.0, with the default send buffer of 212992 bytes, a socket pair takes 278 small datagrams before sendmsg() fails with EAGAIN. All processes share that limit, so the router engine must leave 278 datagrams of the port unread first. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 Reviewed-by: DeepSeek V4.1 Flash Harness: DeepSeek Harness Reviewed-by: Codex --- CHANGES | 11 ++ docs/changes.xml | 14 +++ src/nxt_unit.c | 53 +++++++++- src/nxt_unit.h | 2 + src/test/nxt_unit_msg_test.c | 198 ++++++++++++++++++++++++++++++++++- 5 files changed, 276 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 1462877fc..2876215bb 100644 --- a/CHANGES +++ b/CHANGES @@ -13,6 +13,17 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 write of a body with a Content-Length to the temporary file now answers 500 instead of closing silently. + *) Bugfix: the router could take a message of an application response + before an earlier message of the same response. With a header over + 1024 bytes and a first body write of 16 to 1024 bytes, it parsed the + body as the header, logged "response buffer too small for fields + count", and answered 503. With a body write over 1024 bytes and a + later smaller one, the client got the two parts in the wrong order, + and nothing was logged. This needed two or more application + processes or threads. The bug had appeared in 1.19.0. The fix is in + libunit: an application of type "external", such as Go or + Node.js, gets it when it is built with the new libunit. + *) Bugfix: the router did not answer "Expect: 100-continue". A client that sent it, as curl and Guzzle do for large uploads, waited for its own timeout before it sent the body. For curl, that timeout is 1 diff --git a/docs/changes.xml b/docs/changes.xml index 863f55cdf..4098aa25f 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -32,6 +32,20 @@ file now answers 500 instead of closing silently. + + +the router could take a message of an application response before an earlier +message of the same response. With a header over 1024 bytes and a first body +write of 16 to 1024 bytes, it parsed the body as the header, logged "response +buffer too small for fields count", and answered 503. With a body write over +1024 bytes and a later smaller one, the client got the two parts in the wrong +order, and nothing was logged. This needed two or more application processes +or threads. The bug had appeared in 1.19.0. The fix is in libunit: an +application of type "external", such as Go or Node.js, gets it when it is +built with the new libunit. + + + the router did not answer "Expect: 100-continue". A client that sent it, as diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 04f44c993..c62d5300f 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -1472,6 +1472,39 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) } +#if (NXT_TESTS) + +/* + * Adds a port with a shared memory queue, as NEW_PORT does for a router + * engine port. libunit owns "out_fd" and "queue" from here on; "queue" is + * a mapping of sizeof(nxt_port_queue_t) bytes. + */ + +int +nxt_unit_test_add_queue_port(nxt_unit_ctx_t *ctx, pid_t pid, uint16_t id, + int out_fd, void *queue) +{ + nxt_unit_port_t new_port, *port; + + nxt_unit_port_id_init(&new_port.id, pid, id); + + new_port.in_fd = -1; + new_port.out_fd = out_fd; + new_port.data = NULL; + + port = nxt_unit_add_port(ctx, &new_port, queue); + if (nxt_slow_path(port == NULL)) { + return NXT_UNIT_ERROR; + } + + nxt_unit_port_release(port); + + return NXT_UNIT_OK; +} + +#endif + + static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx) { @@ -7395,13 +7428,31 @@ nxt_unit_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_int_t rc; nxt_port_msg_t msg; nxt_unit_impl_t *lib; + const nxt_port_msg_t *pm; nxt_unit_port_impl_t *port_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); + + /* + * A message that goes to the socket puts only a READ_SOCKET marker into + * the queue. The marker does not tell who sent it. For each marker, + * the reader takes the next datagram, and that datagram can be from + * another process (nxt_port_queue_read_handler()). Thus a datagram can + * be read before a queue message that its sender sent earlier. A + * datagram is never read after a queue message that its sender sent + * later. So a message of a request goes into the queue only if it is + * the last message of the request. A message of no request (stream 0) + * can also go into the queue, even if "last" is clear. QUIT and + * RPC_READY to the port of a context are such messages. + */ + pm = buf; + if (port_impl->queue != NULL && (oob == NULL || oob->size == 0) - && buf_size <= NXT_PORT_QUEUE_MSG_SIZE) + && buf_size <= NXT_PORT_QUEUE_MSG_SIZE + && buf_size >= sizeof(nxt_port_msg_t) + && (pm->stream == 0 || pm->last)) { rc = nxt_port_queue_send(port_impl->queue, buf, buf_size, ¬ify); if (nxt_slow_path(rc != NXT_OK)) { diff --git a/src/nxt_unit.h b/src/nxt_unit.h index a622174e7..6588c65a9 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -486,6 +486,8 @@ uint8_t nxt_unit_test_ctx_online(nxt_unit_ctx_t *ctx); uint8_t nxt_unit_test_ctx_ready(nxt_unit_ctx_t *ctx); void nxt_unit_test_ctx_set_ready(nxt_unit_ctx_t *ctx, uint8_t val); void nxt_unit_test_ctx_quit_graceful(nxt_unit_ctx_t *ctx); +int nxt_unit_test_add_queue_port(nxt_unit_ctx_t *ctx, pid_t pid, + uint16_t id, int out_fd, void *queue); #endif #if (NXT_TESTS || NXT_FUZZ_BUILD) diff --git a/src/test/nxt_unit_msg_test.c b/src/test/nxt_unit_msg_test.c index 612873018..11bdc0dd5 100644 --- a/src/test/nxt_unit_msg_test.c +++ b/src/test/nxt_unit_msg_test.c @@ -41,10 +41,13 @@ typedef enum { NXT_UNIT_MSG_TEST_GROUP_DUP, NXT_UNIT_MSG_TEST_RAISE_COUNT, NXT_UNIT_MSG_TEST_UPGRADE, + NXT_UNIT_MSG_TEST_ORDER, } nxt_unit_msg_test_mode_t; #define NXT_UNIT_MSG_TEST_ROUTER_PORT 2 +#define NXT_UNIT_MSG_TEST_ENGINE_PORT 5 +#define NXT_UNIT_MSG_TEST_FIELDS 32 #define NXT_UNIT_MSG_TEST_STREAM 7 #define NXT_UNIT_MSG_TEST_BODY "ab" @@ -62,6 +65,14 @@ static pid_t nxt_unit_msg_test_pid; static nxt_unit_ctx_t *nxt_unit_msg_test_ctx; static nxt_unit_ctx_t *nxt_unit_msg_test_follower; static nxt_port_mmap_header_t *nxt_unit_msg_test_seg0; +static uint16_t nxt_unit_msg_test_reply_port + = NXT_UNIT_MSG_TEST_ROUTER_PORT; + +/* What libunit sent to the socket of the engine port, in order. */ +static nxt_port_msg_t nxt_unit_msg_test_dgrams[16]; +static size_t nxt_unit_msg_test_ndgrams; + +static int nxt_unit_msg_test_respond(nxt_unit_request_info_t *req); static int nxt_unit_msg_test_send_records_to(nxt_unit_ctx_t *ctx, const nxt_port_mmap_msg_t *records, size_t nrecords, size_t tail); @@ -143,6 +154,14 @@ nxt_unit_msg_test_handler(nxt_unit_request_info_t *req) /* The request now waits for frames; the close finishes it. */ return; + + case NXT_UNIT_MSG_TEST_ORDER: + nxt_unit_msg_test_handler_ok = + nxt_unit_msg_test_respond(req) == NXT_UNIT_OK; + + nxt_unit_request_done(req, NXT_UNIT_OK); + + return; } nxt_unit_request_done(req, NXT_UNIT_ERROR); @@ -184,6 +203,15 @@ nxt_unit_msg_test_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, msg = buf; + if (port->id.id == NXT_UNIT_MSG_TEST_ENGINE_PORT + && buf_size >= sizeof(nxt_port_msg_t) + && msg->type != _NXT_PORT_MSG_READ_QUEUE + && nxt_unit_msg_test_ndgrams < nxt_nitems(nxt_unit_msg_test_dgrams)) + { + memcpy(&nxt_unit_msg_test_dgrams[nxt_unit_msg_test_ndgrams++], msg, + sizeof(nxt_port_msg_t)); + } + /* * While one context asks the router for a segment, another one reads * a record for the same segment and parks behind it. @@ -664,7 +692,7 @@ nxt_unit_msg_test_send_chunk(uint8_t type, uint8_t last, nxt_chunk_id_t chunk, /* The router port was registered with the pid libunit started with. */ msg.stream = NXT_UNIT_MSG_TEST_STREAM; msg.pid = nxt_unit_msg_test_pid; - msg.reply_port = NXT_UNIT_MSG_TEST_ROUTER_PORT; + msg.reply_port = nxt_unit_msg_test_reply_port; msg.type = type; msg.last = last; msg.mmap = 1; @@ -907,6 +935,169 @@ nxt_unit_msg_test_preread_race_case(void *data) } +/* + * A Drupal REST error: a header of many fields, over 1024 bytes, goes into + * shared memory; a short JSON body goes as a plain message. + */ +static int +nxt_unit_msg_test_respond(nxt_unit_request_info_t *req) +{ + int i, rc; + char name[] = "X-Test-00"; + + static const char value[] = "config:field.storage.entity_test.field_test"; + static const char body[] = "{\"message\":\"No route found\"}"; + + rc = nxt_unit_response_init(req, 404, NXT_UNIT_MSG_TEST_FIELDS, + NXT_UNIT_MSG_TEST_FIELDS + * (sizeof(name) + sizeof(value))); + + for (i = 0; rc == NXT_UNIT_OK && i < NXT_UNIT_MSG_TEST_FIELDS; i++) { + name[7] = '0' + i / 10; + name[8] = '0' + i % 10; + + rc = nxt_unit_response_add_field(req, name, nxt_length(name), + value, nxt_length(value)); + } + + if (rc == NXT_UNIT_OK) { + rc = nxt_unit_response_send(req); + } + + if (rc == NXT_UNIT_OK) { + rc = nxt_unit_response_write(req, body, nxt_length(body)); + } + + return rc; +} + + +/* + * A router engine port has a shared memory queue. A message that goes to + * the socket puts only a READ_SOCKET marker into the queue, and for each + * marker the router reads the next datagram, from whichever process sent + * it (nxt_port_queue_read_handler()). Process "Y" puts its marker in + * before this request and sends its datagram after it, as if Y was + * preempted between the two. The router must still get the messages of + * the request in the order libunit sent them: ack, header, body, last. + * libunit put the header into the queue and the body on the socket, so the + * router read the body for Y's marker and parsed it as the header: + * "response buffer too small for fields count: 574235237" and a 503. + */ +static int +nxt_unit_msg_test_order_case(void *data) +{ + int rc, pair[2], notify; + char seq[16]; + void *queue; + size_t n, next; + ssize_t size; + uint8_t qmsg[NXT_PORT_QUEUE_MSG_SIZE]; + nxt_port_msg_t m; + + if (socketpair(AF_UNIX, SOCK_DGRAM, 0, pair) == -1) { + perror("socketpair"); + return 1; + } + + queue = mmap(NULL, sizeof(nxt_port_queue_t), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (queue == MAP_FAILED) { + perror("mmap"); + return 1; + } + + nxt_port_queue_init(queue); + + rc = nxt_unit_test_add_queue_port(nxt_unit_msg_test_ctx, + nxt_unit_msg_test_pid, + NXT_UNIT_MSG_TEST_ENGINE_PORT, pair[0], + queue); + if (rc != NXT_UNIT_OK) { + printf("unit msg test: engine port not added\n"); + return 2; + } + + /* Y's marker. */ + qmsg[0] = _NXT_PORT_MSG_READ_SOCKET; + + if (nxt_port_queue_send(queue, qmsg, 1, ¬ify) != NXT_OK) { + return 3; + } + + nxt_unit_msg_test_mode = NXT_UNIT_MSG_TEST_ORDER; + nxt_unit_msg_test_reply_port = NXT_UNIT_MSG_TEST_ENGINE_PORT; + nxt_unit_msg_test_ndgrams = 0; + + rc = nxt_unit_msg_test_send_request(2, NULL); + + if (rc != NXT_UNIT_OK || nxt_unit_msg_test_handler_calls != 1 + || !nxt_unit_msg_test_handler_ok + || nxt_unit_msg_test_ndgrams == nxt_nitems(nxt_unit_msg_test_dgrams)) + { + printf("unit msg test: response failed: %d\n", rc); + return 4; + } + + /* Y's datagram. */ + memset(&m, 0, sizeof(m)); + + m.stream = NXT_UNIT_MSG_TEST_STREAM + 1; + m.type = _NXT_PORT_MSG_DATA; + + nxt_unit_msg_test_dgrams[nxt_unit_msg_test_ndgrams++] = m; + + /* The router reads the queue, and the next datagram for each marker. */ + n = 0; + next = 0; + + while (n < sizeof(seq) - 1) { + size = nxt_port_queue_recv(queue, qmsg); + if (size < 0) { + break; + } + + if (size == 1 && qmsg[0] == _NXT_PORT_MSG_READ_SOCKET) { + if (next == nxt_unit_msg_test_ndgrams) { + printf("unit msg test: a marker without a datagram\n"); + return 5; + } + + m = nxt_unit_msg_test_dgrams[next++]; + + } else if (size >= (ssize_t) sizeof(nxt_port_msg_t)) { + memcpy(&m, qmsg, sizeof(nxt_port_msg_t)); + + } else { + printf("unit msg test: a %d-byte queue message\n", (int) size); + return 6; + } + + if (m.stream != NXT_UNIT_MSG_TEST_STREAM) { + continue; + } + + seq[n++] = (m.type == _NXT_PORT_MSG_REQ_HEADERS_ACK) ? 'A' + : m.last ? 'L' + : m.mmap ? 'H' : 'B'; + } + + seq[n] = '\0'; + + if (strcmp(seq, "AHBL") != 0) { + printf("unit msg test: the router got \"%s\", expected \"AHBL\" " + "(ack, header, body, last)\n", seq); + + /* The child ends with _exit(), which does not flush stdout. */ + fflush(stdout); + + return 7; + } + + return NXT_UNIT_MSG_TEST_RC(NXT_UNIT_OK); +} + + /* A request upgraded to a websocket, then one frame in shared memory. */ static int nxt_unit_msg_test_websocket_case(void *data) @@ -1119,6 +1310,11 @@ main(void) nxt_unit_msg_test_preread_race_case, NULL, NXT_UNIT_OK); + nxt_unit_msg_test_in_child("response messages keep their order through " + "the queue and the socket", + nxt_unit_msg_test_order_case, NULL, + NXT_UNIT_OK); + nxt_unit_msg_test_in_child("masked websocket frame in shared memory", nxt_unit_msg_test_websocket_case, (void *) &whole_frame, NXT_UNIT_OK); From 34bf22e2f8ff08187fe6590cfbe50094a00bacd7 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 00:13:43 +0200 Subject: [PATCH 349/391] http: state how the rate floors work with a zero timeout When body_read_timeout or send_timeout is 0, the router arms no gap timer for it, so the grace time of body_min_rate or send_min_rate is 0 too. A new window starts after each check. A body read is checked against the time since the previous read, or since the start of the body read state. A write is checked against the time since the previous write, or since the start of the send period if that is later. The floor is not ignored: it is then the only limit on a slow client. But it does not stop a client that stops fully, because the check runs only when a read or a write completes. The docs said "each read is checked. The value 0 disables the check" right after "If body_read_timeout is 0". That could be read as the timeout turning the floor off. They now say what each read or write is checked against, and that a timeout of 0 also turns off the timer that stops a client that sends or reads nothing. They say "A body_min_rate of 0 disables the check", and the same for send_min_rate. A comment above nxt_h1p_rate_too_low() says the same. The C change is a comment only. The tests of both floors now also run with the related timeout at 0. A floor made inert at a grace time of 0 fails them. The comment of the gap test names the assertion that catches a router that counts the gap. This follows https://github.com/freeunitorg/freeunit/pull/508, which was merged before the review of its head 3d881276 was addressed. Co-Authored-By: Claude Opus 5.5 Reviewed-by: DeepSeek V4.1 Flash Harness: DeepSeek Harness Reviewed-by: Claude Fable 5.1 --- docs/unit-openapi.yaml | 17 +++++++++++---- src/nxt_h1proto.c | 10 +++++++++ test/test_settings.py | 48 ++++++++++++++++++++++++++---------------- 3 files changed, 53 insertions(+), 22 deletions(-) diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 94f00a08a..1feed29cf 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -8055,8 +8055,12 @@ components: counts the body bytes that it reads after the request header. The read that completes the body is not checked, because the body is then complete. If `body_read_timeout` is 0, there is - no grace time and each read is checked. The value 0 disables - the check. Use the + no grace time. Each read is then checked against the time + since the previous read, or since Unit started to read the + body. A `body_read_timeout` of 0 also turns off the timer that + stops a client that sends nothing. The check runs only when + a read completes, so a client that stops sending is then never + stopped. A `body_min_rate` of 0 disables the check. Use the value 256 to stop slow clients (\"slow POST\")." default: 0 @@ -8121,8 +8125,13 @@ components: measured rate is near the real download rate of the client. A floor above the bandwidth of honest slow clients stops their downloads. Use a small value, for example 16384. If - `send_timeout` is 0, there is no grace time and each write is - checked. The value 0 disables the check. WebSocket + `send_timeout` is 0, there is no grace time. Each write is + then checked against the time since the previous write, but + only the time when response data waits for the client is + counted. A `send_timeout` of 0 also turns off the timer that + stops a client that reads nothing. The check runs only when + a write completes, so a client that stops reading is then never + stopped. A `send_min_rate` of 0 disables the check. WebSocket connections do not use this check." default: 0 diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 0da22569b..5cc289f21 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -2053,6 +2053,16 @@ nxt_h1p_conn_request_error(nxt_task_t *task, void *obj, void *data) * short network stops occur. A total time limit is not used, because it * also stops honest large transfers on slow links. * + * A timeout of 0 turns the gap timer off: nxt_conn_timer() arms no timer + * for the value 0. Then nothing stops a client that stops fully, because + * the floor check needs a read or a write. The grace time is 0 too, so + * a new window starts after each check. A body read is checked against + * the time since the previous read, or since the start of the body read + * state. A write is checked against the time since the previous write, + * or since the start of the send period if that is later. Thus the time + * between send periods is not counted. A check in the same millisecond + * passes. + * * The check is "bytes * 1000 < rate * msec" in 64-bit integers. The * validator keeps the rate at or below 2^31 - 1, and the check limits * msec to 2^31 - 1. Thus the product cannot overflow. diff --git a/test/test_settings.py b/test/test_settings.py index a37ad8359..7bbf3502a 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -390,15 +390,18 @@ def min_rate_reset(): client.conf_delete(f'settings/http/{name}') -def test_settings_body_min_rate_slow_body(): +@pytest.mark.parametrize('timeout', [2, 0]) +def test_settings_body_min_rate_slow_body(timeout): # A client sends the body 1 byte each second. Each byte starts the # body_read_timeout gap timer again, so only the rate floor can stop # this client. The floor check starts after the grace time (equal to # body_read_timeout, 2 s). The first body read after the grace time - # must give 408. + # must give 408. With body_read_timeout 0 there is no gap timer and + # no grace time, and each read is checked against the time since the + # previous read. The first slow read must give 408. client.load('empty') - min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + min_rate_conf({'body_read_timeout': timeout, 'body_min_rate': 256}) try: sock = socket.create_connection(('127.0.0.1', 8080)) @@ -508,14 +511,16 @@ def test_settings_body_min_rate_burst_then_slow(): min_rate_reset() -def test_settings_body_min_rate_normal_body(): - # A body that is sent faster than the floor must not be stopped. - # The second request on the same keep-alive connection comes after - # an idle time that is longer than the grace time. It must start - # with new rate state. +@pytest.mark.parametrize('timeout', [2, 0]) +def test_settings_body_min_rate_normal_body(timeout): + # A body that is sent faster than the floor must not be stopped, also + # with body_read_timeout 0, when each read is checked. The second + # request on the same keep-alive connection comes after an idle time + # that is longer than the grace time. It must start with new rate + # state. client.load('empty') - min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + min_rate_conf({'body_read_timeout': timeout, 'body_min_rate': 256}) def req(sock, close): sock.sendall( @@ -672,11 +677,14 @@ def test_settings_body_min_rate_expect_wait(): min_rate_reset() -def test_settings_send_min_rate_slow_read(system, wait_for_record): +@pytest.mark.parametrize('timeout', [3, 0]) +def test_settings_send_min_rate_slow_read(timeout, system, wait_for_record): # A client reads a large response slower than send_min_rate, but fast # enough that the router can write some data before each send_timeout. # Each write starts the send_timeout gap timer again, so only the rate - # floor can stop this client. + # floor can stop this client. With send_timeout 0 there is no gap + # timer and no grace time, and each write is checked against the time + # since the previous write. # # The router counts the bytes that the kernel accepts. On loopback the # kernel send buffer grows to some MiB, and the kernel signals a write @@ -698,7 +706,7 @@ def test_settings_send_min_rate_slow_read(system, wait_for_record): client.load('body_generate') - min_rate_conf({'send_timeout': 3, 'send_min_rate': 8388608}) + min_rate_conf({'send_timeout': timeout, 'send_min_rate': 8388608}) try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -761,13 +769,15 @@ def test_settings_send_min_rate_slow_read(system, wait_for_record): min_rate_reset() -def test_settings_send_min_rate_fast_read(): - # A client that reads at full speed must get all the data. Two - # responses on one keep-alive connection check that the second - # response starts with new rate state. +@pytest.mark.parametrize('timeout', [2, 0]) +def test_settings_send_min_rate_fast_read(timeout): + # A client that reads at full speed must get all the data, also with + # send_timeout 0, when each write is checked. Two responses on one + # keep-alive connection check that the second response starts with + # new rate state. client.load('body_generate') - min_rate_conf({'send_timeout': 2, 'send_min_rate': 256}) + min_rate_conf({'send_timeout': timeout, 'send_min_rate': 256}) try: for _ in range(2): @@ -856,7 +866,9 @@ def test_settings_send_min_rate_gap_longer_than_grace(search_in_file): # write would see about 2000 bytes in 2.5 s, below the floor. That # write sends the last data, so the client still gets the full body, # but the router closes the connection. Thus the test sends a second - # request on the same connection. + # request on the same connection. On a router that counts the gap, + # that request finds the connection closed ('gap header'), and this + # assertion fails before the log search. client.load('send_bursts') min_rate_conf({'send_timeout': 2, 'send_min_rate': 1048576}) From 92adbb221681582e94c5f908252cb37d44886924 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 00:43:58 +0200 Subject: [PATCH 350/391] tests: check the body floor record, and test chunked and TLS bodies Each test that expects body_min_rate to stop a client now also waits for the record "client body rate is less than body_min_rate" in unit.log. The send tests already did this. Before, a close for another reason within 8 s also passed. A router that sends the 408 without the record fails 6 tests. Two new tests run the slow body test with other inputs. One sends a chunked body, a chunk of 1 byte each second. A floor that skips chunked bodies fails it. The other sends the body over TLS. A floor that is off fails it. It skips when Unit is built without OpenSSL. The loop of the slow body tests is now one function, min_rate_slow_body(). The sanitize (python) leg in .github/workflows/sanitize.yml now runs test/test_settings.py. That leg is configured with --openssl, so it also runs the TLS test. The openapi descriptions of body_min_rate and send_min_rate, and the entry in CHANGES and docs/changes.xml, now say that only HTTP/1 connections use the floors. The entry is in the 1.36.2 block, which is not released. https://github.com/freeunitorg/docs/pull/47 says the same. This follows https://github.com/freeunitorg/freeunit/pull/508. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/sanitize.yml | 6 + CHANGES | 28 ++--- docs/changes.xml | 7 +- docs/unit-openapi.yaml | 6 +- test/test_settings.py | 223 +++++++++++++++++++++------------ 5 files changed, 170 insertions(+), 100 deletions(-) diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 77702811f..8dacaa57b 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -143,6 +143,11 @@ jobs: # test_state_store forks a store child from main and stops main # while the child runs (#516); it needs --restart, and this is # the leg that runs that path under ASan. + # + # test_settings runs the body_min_rate and send_min_rate floors. + # A floor closes the connection from a read or a write handler, + # in the middle of a body or a response. It also has a TLS case, + # which this leg runs because it is configured with --openssl. tests: >- test/test_graceful_reload.py test/test_state_store.py @@ -154,6 +159,7 @@ jobs: test/test_conn_recycle.py test/test_app_lifecycle.py test/test_configuration.py + test/test_settings.py - runtime: php apt: "" configure: php diff --git a/CHANGES b/CHANGES index 2876215bb..e306ca8e8 100644 --- a/CHANGES +++ b/CHANGES @@ -252,20 +252,20 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 Content-Length. *) Feature: the "body_min_rate" and "send_min_rate" options in - "settings/http" set a minimum client rate in bytes per second. - The body_read_timeout and send_timeout timers start again after - each read or write, so a client that sends or reads one byte - before each timeout kept a connection for ever. After a grace - time equal to the related timeout, a client below the rate gets - 408 (body) or the connection closes (response). The rate is - checked for each window of at least the grace time, so bytes sent - early give no credit for a slow transfer later. The default is 0, - which disables the check. For body_min_rate, 256 is a good value. - send_min_rate counts the bytes that the kernel accepts into the - socket send buffer. The kernel sizes this buffer to the link, so - the rate is near the real download rate of the client. A floor - above the bandwidth of honest slow clients stops their downloads; - use a small value, for example 16384. + "settings/http" set a minimum client rate in bytes per second on + HTTP/1 connections. The body_read_timeout and send_timeout timers + start again after each read or write, so a client that sends or + reads one byte before each timeout kept a connection for ever. + After a grace time equal to the related timeout, a client below + the rate gets 408 (body) or the connection closes (response). The + rate is checked for each window of at least the grace time, so + bytes sent early give no credit for a slow transfer later. The + default is 0, which disables the check. For body_min_rate, 256 is + a good value. send_min_rate counts the bytes that the kernel + accepts into the socket send buffer. The kernel sizes this buffer + to the link, so the rate is near the real download rate of the + client. A floor above the bandwidth of honest slow clients stops + their downloads; use a small value, for example 16384. *) Feature: the --hardening=[off|default|strict] configure option adds compiler and linker hardening flags. Each flag is probed, and a diff --git a/docs/changes.xml b/docs/changes.xml index 4098aa25f..09a4d8d5e 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -369,9 +369,10 @@ it carried chunked together with Content-Length. the "body_min_rate" and "send_min_rate" options in "settings/http" set -a minimum client rate in bytes per second. The body_read_timeout and -send_timeout timers start again after each read or write, so a client that -sends or reads one byte before each timeout kept a connection for ever. +a minimum client rate in bytes per second on HTTP/1 connections. The +body_read_timeout and send_timeout timers start again after each read or +write, so a client that sends or reads one byte before each timeout kept +a connection for ever. After a grace time equal to the related timeout, a client below the rate gets 408 (body) or the connection closes (response). The rate is checked for each window of at least the grace time, so bytes sent early give no credit for diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 1feed29cf..14d0b6276 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -8046,7 +8046,8 @@ components: minimum: 0 maximum: 2147483647 description: "Minimum rate of the client request body, in bytes - per second. The check starts after a grace time. The grace + per second. Only HTTP/1 connections use this check. The check + starts after a grace time. The grace time is equal to `body_read_timeout`. After the grace time, if the client sent fewer bytes than `body_min_rate` multiplied by the elapsed seconds, Unit sends 408 and closes the @@ -8114,7 +8115,8 @@ components: minimum: 0 maximum: 2147483647 description: "Minimum rate at which the client reads the response, - in bytes per second. Unit measures only the time when response + in bytes per second. Only HTTP/1 connections use this check. + Unit measures only the time when response data waits for the client. The check starts after a grace time. The grace time is equal to `send_timeout`. After the grace time, if the client accepted fewer bytes than `send_min_rate` diff --git a/test/test_settings.py b/test/test_settings.py index 7bbf3502a..44a13e72a 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -1,11 +1,14 @@ import re import socket +import ssl import subprocess import time import pytest from unit.applications.lang.python import ApplicationPython +from unit.applications.tls import ApplicationTLS +from unit.option import option prerequisites = {'modules': {'python': 'any'}} @@ -386,12 +389,49 @@ def min_rate_reset(): 'body_min_rate', 'send_timeout', 'send_min_rate', + 'chunked_transform', ): client.conf_delete(f'settings/http/{name}') +BODY_RATE_RECORD = r'client body rate is less than body_min_rate' + + +def min_rate_slow_body(sock, part, limit): + # Sends "part" about once each second for at most "limit" seconds, and + # reads what the router sends back. Returns (closed, data, elapsed). + sock.settimeout(1) + + start = time.monotonic() + data = b'' + closed = False + + while time.monotonic() - start < limit: + try: + sock.sendall(part) + except OSError: + closed = True + break + + try: + chunk = sock.recv(4096) + except socket.timeout: + continue + except OSError: + closed = True + break + + if not chunk: + closed = True + break + + data += chunk + + return closed, data, time.monotonic() - start + + @pytest.mark.parametrize('timeout', [2, 0]) -def test_settings_body_min_rate_slow_body(timeout): +def test_settings_body_min_rate_slow_body(timeout, wait_for_record): # A client sends the body 1 byte each second. Each byte starts the # body_read_timeout gap timer again, so only the rate floor can stop # this client. The floor check starts after the grace time (equal to @@ -413,32 +453,7 @@ def test_settings_body_min_rate_slow_body(timeout): b'Connection: close\r\n\r\n' ) - start = time.monotonic() - data = b'' - closed = False - - while time.monotonic() - start < 12: - try: - sock.sendall(b'x') - except OSError: - closed = True - break - - try: - chunk = sock.recv(4096) - except socket.timeout: - continue - except OSError: - closed = True - break - - if not chunk: - closed = True - break - - data += chunk - - elapsed = time.monotonic() - start + closed, data, elapsed = min_rate_slow_body(sock, b'x', 12) sock.close() # The router closes the connection after the 408 response. The @@ -447,12 +462,102 @@ def test_settings_body_min_rate_slow_body(timeout): assert closed or data, 'slow body stopped' assert data == b'' or data.startswith(b'HTTP/1.1 408'), 'slow body 408' assert elapsed < 8, 'slow body stopped in time' + assert wait_for_record(BODY_RATE_RECORD) is not None, 'slow body log' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_chunked(wait_for_record): + # The same slow client as in the slow body test, with a chunked body. + # The client sends a chunk of 1 byte each second. The chunked body is + # read in the same function as a body with Content-Length, but the + # chunk parser runs there first. The floor counts the chunk framing + # too: 6 bytes each second, which is much less than 256. + client.load('empty') + + min_rate_conf( + { + 'body_read_timeout': 2, + 'body_min_rate': 256, + 'chunked_transform': True, + } + ) + + try: + sock = socket.create_connection(('127.0.0.1', 8080)) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Transfer-Encoding: chunked\r\n' + b'Connection: close\r\n\r\n' + ) + + closed, data, elapsed = min_rate_slow_body(sock, b'1\r\nx\r\n', 12) + sock.close() + + assert closed or data, 'chunked body stopped' + assert data == b'' or data.startswith( + b'HTTP/1.1 408' + ), 'chunked body 408' + assert elapsed < 8, 'chunked body stopped in time' + assert wait_for_record(BODY_RATE_RECORD) is not None, 'chunked log' + + finally: + min_rate_reset() + + +def test_settings_body_min_rate_tls(wait_for_record): + # The same slow client as in the slow body test, on a TLS listener. + # The floor counts the bytes that the TLS layer gives to the router, + # so 1 byte each second is still below 256. + if not option.available['modules'].get('openssl'): + pytest.skip('requires openssl') + + client.load('empty') + + ApplicationTLS().certificate() + + assert 'success' in client.conf( + { + "pass": "applications/empty", + "tls": {"certificate": "default"}, + }, + 'listeners/*:8080', + ) + + min_rate_conf({'body_read_timeout': 2, 'body_min_rate': 256}) + + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + try: + sock = context.wrap_socket( + socket.create_connection(('127.0.0.1', 8080)) + ) + sock.sendall( + b'POST / HTTP/1.1\r\n' + b'Host: localhost\r\n' + b'Content-Length: 10000\r\n' + b'Connection: close\r\n\r\n' + ) + + closed, data, elapsed = min_rate_slow_body(sock, b'x', 12) + sock.close() + + assert closed or data, 'TLS slow body stopped' + assert data == b'' or data.startswith( + b'HTTP/1.1 408' + ), 'TLS slow body 408' + assert elapsed < 8, 'TLS slow body stopped in time' + assert wait_for_record(BODY_RATE_RECORD) is not None, 'TLS log' finally: min_rate_reset() -def test_settings_body_min_rate_burst_then_slow(): +def test_settings_body_min_rate_burst_then_slow(wait_for_record): # A client sends half of the body at once, then 1 byte each second. # With one average over the whole body, the burst would give credit # for about 195 s at 256 bytes per second. The rate is checked for @@ -473,32 +578,7 @@ def test_settings_body_min_rate_burst_then_slow(): ) sock.sendall(b'x' * 50000) - start = time.monotonic() - data = b'' - closed = False - - while time.monotonic() - start < 20: - try: - sock.sendall(b'x') - except OSError: - closed = True - break - - try: - chunk = sock.recv(4096) - except socket.timeout: - continue - except OSError: - closed = True - break - - if not chunk: - closed = True - break - - data += chunk - - elapsed = time.monotonic() - start + closed, data, elapsed = min_rate_slow_body(sock, b'x', 20) sock.close() assert closed or data, 'burst then slow body stopped' @@ -506,6 +586,9 @@ def test_settings_body_min_rate_burst_then_slow(): b'HTTP/1.1 408' ), 'burst then slow body 408' assert elapsed < 12, 'burst then slow body stopped in time' + assert ( + wait_for_record(BODY_RATE_RECORD) is not None + ), 'burst then slow body log' finally: min_rate_reset() @@ -587,7 +670,7 @@ def min_rate_expect(length): return sock -def test_settings_body_min_rate_expect_slow_body(): +def test_settings_body_min_rate_expect_slow_body(wait_for_record): # After a 100 (Continue), the body read state starts in another # function. The floor must be active there too. Otherwise a client # can avoid the floor with "Expect: 100-continue". The client sends @@ -600,32 +683,7 @@ def test_settings_body_min_rate_expect_slow_body(): sock = min_rate_expect(10000) sock.settimeout(1) - start = time.monotonic() - data = b'' - closed = False - - while time.monotonic() - start < 12: - try: - sock.sendall(b'x') - except OSError: - closed = True - break - - try: - chunk = sock.recv(4096) - except socket.timeout: - continue - except OSError: - closed = True - break - - if not chunk: - closed = True - break - - data += chunk - - elapsed = time.monotonic() - start + closed, data, elapsed = min_rate_slow_body(sock, b'x', 12) sock.close() assert closed or data, 'slow body after 100 stopped' @@ -633,6 +691,9 @@ def test_settings_body_min_rate_expect_slow_body(): b'HTTP/1.1 408' ), 'slow body after 100 408' assert elapsed < 8, 'slow body after 100 stopped in time' + assert ( + wait_for_record(BODY_RATE_RECORD) is not None + ), 'slow body after 100 log' finally: min_rate_reset() From 31df46befd6a29592dbeb0eb2039774615df739d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 3 Oct 2026 12:40:05 +0200 Subject: [PATCH 351/391] wasm: read the guest memory base again after the guest runs The host read the base address of the linear memory once, in nxt_wasmtime_init_memory(), and used it for every later access. A memory.grow can move the memory. Wasmtime 30 and later accept a 64-bit memory by default, and such a memory has no 4 GiB limit. When it grows past the 4 GiB that wasmtime reserves on a 64-bit host, wasmtime maps a new region, copies the data and unmaps the old region. The host then read and wrote the unmapped region, and the worker died on SIGSEGV. The host now refuses a 64-bit memory before it calls the malloc handler. The lookup of the "memory" export also checks that the export is a memory. The host reads the base again after each call of the request handler or a hook, and in the send_response and send_headers imports. On a 32-bit host wasmtime reserves only 10 MiB, so a wasm32 memory can move as well. A new test loads a guest with a 64-bit memory. The router answers 503. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 7 ++++ docs/changes.xml | 10 ++++++ src/wasm/nxt_rt_wasmtime.c | 68 +++++++++++++++++++++++++++++++++-- src/wasm/nxt_wasm.c | 3 ++ src/wasm/nxt_wasm.h | 4 +-- test/test_wasm_memory64.py | 60 +++++++++++++++++++++++++++++++ test/wasm/memory64/memory64.c | 53 +++++++++++++++++++++++++++ 7 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 test/test_wasm_memory64.py create mode 100644 test/wasm/memory64/memory64.c diff --git a/CHANGES b/CHANGES index e306ca8e8..7e598c43f 100644 --- a/CHANGES +++ b/CHANGES @@ -115,6 +115,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 and handed out stream 0, which callers read as a failure. The registration stayed. Stream 0 is now skipped. + *) Security: the wasm module read the base address of the linear + memory of the guest only once, at startup. A guest with a 64-bit + memory could grow it past 4 GiB, and wasmtime then moved it. The + module then read and wrote the old address, and the worker crashed. + The module now refuses a 64-bit memory. It also reads the base + address again after the guest runs. + *) Bugfix: a keep-alive TLS connection that was idle across a reconfiguration could use the TLS settings of the old configuration after they were freed, when it was closed. The connection now keeps diff --git a/docs/changes.xml b/docs/changes.xml index 09a4d8d5e..83cde03a4 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -229,6 +229,16 @@ chunk marker that the application can clear. All of these are now checked. + + +the wasm module read the base address of the linear memory of the guest only +once, at startup. A guest with a 64-bit memory could grow it past 4 GiB, and +wasmtime then moved it. The module then read and wrote the old address, and +the worker crashed. The module now refuses a 64-bit memory. It also reads +the base address again after the guest runs. + + + a keep-alive TLS connection that was idle across a reconfiguration could diff --git a/src/wasm/nxt_rt_wasmtime.c b/src/wasm/nxt_rt_wasmtime.c index 570a6298a..81b478e22 100644 --- a/src/wasm/nxt_rt_wasmtime.c +++ b/src/wasm/nxt_rt_wasmtime.c @@ -58,6 +58,19 @@ nxt_wasmtime_err_msg(wasmtime_error_t *error, wasm_trap_t *trap, } +/* + * memory.grow can move the linear memory to a new address. Read the base + * again after the guest has run, and before an import function reads or + * writes the memory. + */ +static void +nxt_wasmtime_update_baddr(nxt_wasm_ctx_t *ctx, wasmtime_context_t *wctx) +{ + ctx->baddr = wasmtime_memory_data(wctx, &nxt_wasmtime_ctx.memory) + + ctx->baddr_off; +} + + static wasm_trap_t * nxt_wasm_get_init_mem_size(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, @@ -85,6 +98,8 @@ nxt_wasm_send_response(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, wasmtime_val_t *results, size_t nresults) { + nxt_wasmtime_update_baddr(env, wasmtime_caller_context(caller)); + nxt_wasm_do_send_response(env, args[0].of.i32); return NULL; @@ -96,6 +111,8 @@ nxt_wasm_send_headers(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, wasmtime_val_t *results, size_t nresults) { + nxt_wasmtime_update_baddr(env, wasmtime_caller_context(caller)); + nxt_wasm_do_send_headers(env, args[0].of.i32); return NULL; @@ -116,7 +133,7 @@ nxt_wasm_set_resp_status(void *env, wasmtime_caller_t *caller, static void -nxt_wasmtime_execute_hook(const nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook) +nxt_wasmtime_execute_hook(nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook) { const char *name = ctx->fh[hook].func_name; wasm_trap_t *trap = NULL; @@ -129,6 +146,9 @@ nxt_wasmtime_execute_hook(const nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook) } error = wasmtime_func_call(rt_ctx->ctx, func, NULL, 0, NULL, 0, &trap); + + nxt_wasmtime_update_baddr(ctx, rt_ctx->ctx); + if (error != NULL || trap != NULL) { nxt_wasmtime_err_msg(error, trap, "failed to call hook function [%s]", name); @@ -137,7 +157,7 @@ nxt_wasmtime_execute_hook(const nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook) static int -nxt_wasmtime_execute_request(const nxt_wasm_ctx_t *ctx) +nxt_wasmtime_execute_request(nxt_wasm_ctx_t *ctx) { int i = 0; wasm_trap_t *trap = NULL; @@ -151,6 +171,9 @@ nxt_wasmtime_execute_request(const nxt_wasm_ctx_t *ctx) args[i++].of.i32 = ctx->baddr_off; error = wasmtime_func_call(rt_ctx->ctx, func, args, i, results, 1, &trap); + + nxt_wasmtime_update_baddr(ctx, rt_ctx->ctx); + if (error != NULL || trap != NULL) { nxt_wasmtime_err_msg(error, trap, "failed to call function [->wasm_request_handler]" @@ -302,6 +325,42 @@ nxt_wasmtime_wasi_init(const nxt_wasm_ctx_t *ctx) } +/* + * The host passes and reads wasm32 offsets. A 64-bit memory has no 4 GiB + * limit, so it can grow past the address space that wasmtime reserved for + * it, and wasmtime then moves it. Refuse such a memory before the host + * calls the malloc handler. + */ +static int +nxt_wasmtime_check_memory(void) +{ + bool ok, is64; + wasmtime_extern_t item; + wasm_memorytype_t *type; + nxt_wasmtime_ctx_t *rt_ctx = &nxt_wasmtime_ctx; + + ok = wasmtime_linker_get(rt_ctx->linker, rt_ctx->ctx, "", 0, "memory", + strlen("memory"), &item); + if (!ok || item.kind != WASMTIME_EXTERN_MEMORY) { + nxt_wasmtime_err_msg(NULL, NULL, "couldn't get 'memory' from module"); + return -1; + } + + type = wasmtime_memory_type(rt_ctx->ctx, &item.of.memory); + is64 = wasmtime_memorytype_is64(type); + wasm_memorytype_delete(type); + + if (is64) { + nxt_wasmtime_err_msg(NULL, NULL, + "module memory is 64-bit; only a 32-bit memory " + "is supported"); + return -1; + } + + return 0; +} + + static int nxt_wasmtime_init_memory(nxt_wasm_ctx_t *ctx) { @@ -436,6 +495,11 @@ nxt_wasmtime_init(nxt_wasm_ctx_t *ctx) return -1; } + err = nxt_wasmtime_check_memory(); + if (err) { + return -1; + } + err = nxt_wasmtime_init_memory(ctx); if (err) { return -1; diff --git a/src/wasm/nxt_wasm.c b/src/wasm/nxt_wasm.c index 7a7689efa..8c320a5dc 100644 --- a/src/wasm/nxt_wasm.c +++ b/src/wasm/nxt_wasm.c @@ -312,6 +312,9 @@ nxt_wasm_request_handler(nxt_unit_request_info_t *req) offset = sizeof(nxt_wasm_request_t); do { + /* exec_request() read the base again; the memory may have moved. */ + wr = (nxt_wasm_request_t *) nxt_wasm_ctx.baddr; + read_bytes = nxt_min(content_len - content_sent, NXT_WASM_MEM_SIZE - offset); bytes_read = nxt_unit_request_read(req, (uint8_t *)wr + offset, diff --git a/src/wasm/nxt_wasm.h b/src/wasm/nxt_wasm.h index 25b79605c..f301798a9 100644 --- a/src/wasm/nxt_wasm.h +++ b/src/wasm/nxt_wasm.h @@ -127,8 +127,8 @@ struct nxt_wasm_ctx_s { struct nxt_wasm_operations_s { int (*init)(nxt_wasm_ctx_t *ctx); void (*destroy)(const nxt_wasm_ctx_t *ctx); - int (*exec_request)(const nxt_wasm_ctx_t *ctx); - void (*exec_hook)(const nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook); + int (*exec_request)(nxt_wasm_ctx_t *ctx); + void (*exec_hook)(nxt_wasm_ctx_t *ctx, nxt_wasm_fh_t hook); }; extern const nxt_wasm_operations_t nxt_wasm_ops; diff --git a/test/test_wasm_memory64.py b/test/test_wasm_memory64.py new file mode 100644 index 000000000..b3e72a631 --- /dev/null +++ b/test/test_wasm_memory64.py @@ -0,0 +1,60 @@ +from pathlib import Path +import shutil +import subprocess + +from unit.applications.lang.wasm import ApplicationWasm +from unit.check.check_prerequisites import check_prerequisites +from unit.option import option + +prerequisites = {'modules': {'wasm': 'any'}, 'features': {'clang_wasm': True}} + +check_prerequisites(prerequisites) + +client = ApplicationWasm() + +# 1 MiB below the request buffer, then the 32 MiB + 64 KiB that the host +# asks for, rounded up to 36 MiB. +INITIAL_MEMORY = 36 * 1024 * 1024 + + +def build_memory64_guest(): + source = Path(option.test_dir) / 'wasm' / 'memory64' / 'memory64.c' + output = Path(option.temp_dir) / 'memory64.wasm' + + subprocess.check_output( + [ + shutil.which('clang'), + '--target=wasm64-unknown-unknown', + '-O2', + '-nostdlib', + f'-Wl,--no-entry,--initial-memory={INITIAL_MEMORY}', + str(source), + '-o', + str(output), + ], + stderr=subprocess.STDOUT, + ) + + return output + + +def test_wasm_memory64_refused(wait_for_record): + assert 'success' in client.conf( + { + 'listeners': {'*:8080': {'pass': 'applications/memory64'}}, + 'applications': { + 'memory64': { + 'type': 'wasm', + 'processes': {'spare': 0}, + 'module': str(build_memory64_guest()), + 'request_handler': 'request_handler', + 'malloc_handler': 'malloc_handler', + 'free_handler': 'free_handler', + } + }, + } + ) + + assert client.get()['status'] == 503 + + assert wait_for_record(r'module memory is 64-bit') is not None diff --git a/test/wasm/memory64/memory64.c b/test/wasm/memory64/memory64.c new file mode 100644 index 000000000..e2b10702a --- /dev/null +++ b/test/wasm/memory64/memory64.c @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright (C) FreeUnit contributors. */ + +/* + * A guest with a 64-bit linear memory, built without libc or libunit-wasm. + * The host must refuse it when the worker starts. The handlers work, so a + * host that accepts the module answers 200 with "ok". + */ + +typedef unsigned int u32; + +#define IMPORT(name) __attribute__((import_module("env"), import_name(name))) +#define EXPORT(name) __attribute__((export_name(name))) + +/* The request buffer starts above the data and the stack of the guest. */ +#define BUFFER_OFFSET (1024 * 1024) +#define RESPONSE_OFFSET 4096 + +IMPORT("nxt_wasm_send_response") void send_response(u32 offset); +IMPORT("nxt_wasm_response_end") void response_end(void); + +EXPORT("malloc_handler") u32 +malloc_handler(u32 size) +{ + (void) size; + + return BUFFER_OFFSET; +} + +EXPORT("free_handler") void +free_handler(u32 addr) +{ + (void) addr; +} + +EXPORT("request_handler") int +request_handler(u32 addr) +{ + unsigned char *resp; + + resp = (unsigned char *) (unsigned long) addr + RESPONSE_OFFSET; + + /* The layout of nxt_wasm_response_t: a u32 size, then the data. */ + *(u32 *) resp = 3; + resp[4] = 'o'; + resp[5] = 'k'; + resp[6] = '\n'; + + send_response(RESPONSE_OFFSET); + response_end(); + + return 0; +} From 9983351c738db998e874b33bd1c920eb7f404f05 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 20:30:52 +0000 Subject: [PATCH 352/391] http: answer only the latest WebSocket ping while a pong is pending The router queued a PONG for each PING. A client that sent PINGs and did not read the PONGs could make the router keep a PONG in memory for each PING, until the connection closed. RFC 6455 Section 5.5.3 lets an endpoint send a PONG only for the most recent PING when it did not send the PONGs for the earlier PINGs yet. Now the router queues a PONG only when no PONG is queued. While a PONG is queued, the router reads the next PING and writes its PONG into one spare buffer. Each new PING overwrites this buffer. When the queued PONG is sent, its completion handler queues the spare PONG. The router thus keeps at most two PONGs for a connection, and the most recent PING always gets its PONG. The completion handler does not queue the spare PONG when the request failed or is closed, when the last buffer of the response is sent, or when the connection closes. The spare buffer is not queued while it is in use, so the router never changes a buffer that it is sending. This is necessary for TLS, where a write can be retried with the same data only. There is no configuration option: the bound comes from the protocol. Tests: - test_asgi_websockets_ping_pong_pending: a client with a small receive buffer sends 2000 PINGs and does not read. It gets fewer PONGs than PINGs, in order, the last PING gets its PONG, and the connection still echoes a text frame after that. - The 2_10 and 2_11 tests (ASGI, Node.js, Java) now accept PONGs for a subset of the PINGs, in order, that ends with the last PING. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 6 ++ docs/changes.xml | 9 +++ src/nxt_h1proto.h | 2 + src/nxt_h1proto_websocket.c | 117 +++++++++++++++++++++++++-- test/test_asgi_websockets.py | 54 ++++++++++--- test/test_java_websockets.py | 19 ++--- test/test_node_websockets.py | 19 ++--- test/unit/applications/websockets.py | 30 +++++-- 8 files changed, 213 insertions(+), 43 deletions(-) diff --git a/CHANGES b/CHANGES index 7e598c43f..d6ca94ea7 100644 --- a/CHANGES +++ b/CHANGES @@ -147,6 +147,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 deadline stayed armed for every frame. The deadline now stops at the upgrade. + *) Bugfix: a WebSocket client that sent PINGs and did not read the + PONGs could make the router keep a PONG in memory for each PING. + Now, while a PONG is not sent, the router keeps only the PONG for + the most recent PING, and sends it after the queued PONG (RFC 6455 + Section 5.5.3). + *) Bugfix: the router crashed when it validated a configuration that gave "compressors" in the object form. diff --git a/docs/changes.xml b/docs/changes.xml index 83cde03a4..2ba0001c8 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -279,6 +279,15 @@ frame. The deadline now stops at the upgrade. + + +a WebSocket client that sent PINGs and did not read the PONGs could make the +router keep a PONG in memory for each PING. Now, while a PONG is not sent, the +router keeps only the PONG for the most recent PING, and sends it after the +queued PONG (RFC 6455 Section 5.5.3). + + + the router crashed when it validated a configuration that gave diff --git a/src/nxt_h1proto.h b/src/nxt_h1proto.h index 7d16142a9..c4f3819e4 100644 --- a/src/nxt_h1proto.h +++ b/src/nxt_h1proto.h @@ -47,6 +47,7 @@ struct nxt_h1proto_s { uint8_t websocket_cont_expected; /* 1 bit */ uint8_t websocket_closed; /* 1 bit */ uint8_t websocket_text; /* 1 bit */ + uint8_t websocket_pong_queued; /* 1 bit */ uint8_t continue_pending; /* 1 bit */ nxt_h1p_ws_utf8_t websocket_utf8; @@ -72,6 +73,7 @@ struct nxt_h1proto_s { nxt_http_field_t *websocket_key; nxt_h1p_websocket_timer_t *websocket_timer; + nxt_buf_t *websocket_pong_next; nxt_http_request_t *request; nxt_buf_t *buffers; diff --git a/src/nxt_h1proto_websocket.c b/src/nxt_h1proto_websocket.c index 2f5a69192..5e8c90aa0 100644 --- a/src/nxt_h1proto_websocket.c +++ b/src/nxt_h1proto_websocket.c @@ -34,6 +34,10 @@ static void hxt_h1p_send_ws_error(nxt_task_t *task, nxt_http_request_t *r, const nxt_ws_error_t *err, ...); static void nxt_h1p_conn_ws_error_sent(nxt_task_t *task, void *obj, void *data); static void nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data); +static void nxt_h1p_conn_ws_pong_send(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *out); +static void nxt_h1p_conn_ws_pong_sent(nxt_task_t *task, void *obj, + void *data); static size_t nxt_h1p_ws_utf8_seqlen(u_char c); static nxt_int_t nxt_h1p_ws_utf8_begin(const u_char *seq, size_t n); static nxt_int_t nxt_h1p_ws_utf8_validate(nxt_h1p_ws_utf8_t *state, @@ -930,6 +934,8 @@ nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data) { uint8_t payload_len, i; nxt_buf_t *b, *out, *next; + nxt_bool_t deferred; + nxt_h1proto_t *h1p; nxt_http_request_t *r; nxt_websocket_header_t *wsh; uint8_t mask[4]; @@ -937,6 +943,7 @@ nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data) nxt_debug(task, "h1p conn ws pong"); r = obj; + h1p = r->proto.h1; b = r->ws_frame; wsh = (nxt_websocket_header_t *) b->mem.pos; @@ -948,10 +955,42 @@ nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data) b->mem.pos += 4; - out = nxt_http_buf_mem(task, r, 2 + payload_len); - if (nxt_slow_path(out == NULL)) { - nxt_http_request_error_handler(task, r, r->proto.any); - return; + /* + * RFC 6455 Section 5.5.3 lets an endpoint send a PONG only for the most + * recent PING when it did not send the PONGs for the earlier PINGs yet. + * A client that sends PINGs and does not read the PONGs could otherwise + * make the router keep a PONG in memory for each PING. + * + * Thus, while a PONG is queued, the router does not queue a new PONG. + * It keeps the PONG for the most recent PING in a buffer that the next + * PING overwrites, and queues it when the queued PONG is sent. The + * router then keeps at most two PONGs for a connection. + */ + + deferred = h1p->websocket_pong_queued; + + if (deferred) { + out = h1p->websocket_pong_next; + + if (out == NULL) { + /* A control frame payload is 125 bytes or less. */ + out = nxt_buf_mem_alloc(r->mem_pool, 2 + 125, 0); + if (nxt_slow_path(out == NULL)) { + nxt_http_request_error_handler(task, r, r->proto.any); + return; + } + + h1p->websocket_pong_next = out; + } + + nxt_debug(task, "h1p conn ws pong deferred"); + + } else { + out = nxt_http_buf_mem(task, r, 2 + payload_len); + if (nxt_slow_path(out == NULL)) { + nxt_http_request_error_handler(task, r, r->proto.any); + return; + } } out->mem.start[0] = 0; @@ -979,7 +1018,75 @@ nxt_h1p_conn_ws_pong(nxt_task_t *task, void *obj, void *data) r->ws_frame = b; - nxt_http_request_send(task, r, out); + if (!deferred) { + nxt_h1p_conn_ws_pong_send(task, r, out); + } nxt_http_request_ws_frame_start(task, r, r->ws_frame); } + + +static void +nxt_h1p_conn_ws_pong_send(nxt_task_t *task, nxt_http_request_t *r, + nxt_buf_t *out) +{ + out->completion_handler = nxt_h1p_conn_ws_pong_sent; + out->parent = r; + + r->proto.h1->websocket_pong_queued = 1; + + nxt_http_request_send(task, r, out); +} + + +/* + * The queued PONG is sent, or it is discarded. If a PING came in the + * meantime, queue the PONG for the most recent PING now. Do not queue it + * when the request is closed or failed, or when its last buffer is sent: + * no frame can follow the last buffer. + */ + +static void +nxt_h1p_conn_ws_pong_sent(nxt_task_t *task, void *obj, void *data) +{ + nxt_buf_t *b, *next, *out; + nxt_h1proto_t *h1p; + nxt_http_request_t *r; + + b = obj; + r = data; + + nxt_debug(task, "h1p conn ws pong sent"); + + h1p = r->proto.h1; + + if (h1p != NULL) { + h1p->websocket_pong_queued = 0; + + out = h1p->websocket_pong_next; + h1p->websocket_pong_next = NULL; + + if (out != NULL) { + if (r->error || r->last == NULL || h1p->conn->closing) { + nxt_mp_free(r->mem_pool, out); + + } else { + /* The completion handler releases the pool. */ + nxt_mp_retain(r->mem_pool); + + nxt_h1p_conn_ws_pong_send(task, r, out); + } + } + } + + /* The pool can be freed with the last release, and r with it. */ + + do { + next = b->next; + + nxt_mp_free(r->mem_pool, b); + nxt_mp_release(r->mem_pool); + + b = next; + } while (b != NULL); +} diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index 11356fea6..553e864f1 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -1,3 +1,4 @@ +import socket import struct import time @@ -557,22 +558,53 @@ def test_asgi_websockets_2_10__2_11(): _, sock, _ = ws.upgrade() - for i in range(0, 10): - ws.frame_write(sock, ws.OP_PING, f'payload-{i}') + pings = [f'payload-{i}' for i in range(0, 10)] - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload) + + ws.pongs_read(sock, pings) # 2_11 - for i in range(0, 10): - opcode = ws.OP_PING - ws.frame_write(sock, opcode, f'payload-{i}', chopsize=1) + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload, chopsize=1) - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + ws.pongs_read(sock, pings) + + close_connection(sock) + + +def test_asgi_websockets_ping_pong_pending(): + client.load('websockets/mirror') + + # A client with a small receive buffer sends many PINGs and does not + # read. While a PONG is not sent, the router must keep only the PONG + # for the most recent PING (RFC 6455 Section 5.5.3). The connection + # must stay open, and the last PING must get its PONG. + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 4096) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.connect(('127.0.0.1', 8080)) + + _, sock, _ = ws.upgrade(sock=sock) + + pings = [f'{i:05}'.encode() * 25 for i in range(2000)] + + sock.sendall(b''.join(ws.frame_to_send(ws.OP_PING, p) for p in pings)) + + answered = ws.pongs_read(sock, pings) + + assert answered[0] == 0, 'first ping answered' + assert len(answered) < len(pings), 'pings not answered' + + message = 'still open' + + ws.frame_write(sock, ws.OP_TEXT, message) + + frame = ws.frame_read(sock) + check_frame(frame, True, ws.OP_TEXT, message) close_connection(sock) diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index e56b709bb..78823471e 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -496,22 +496,19 @@ def test_java_websockets_2_10__2_11(): _, sock, _ = ws.upgrade() - for i in range(0, 10): - ws.frame_write(sock, ws.OP_PING, f'payload-{i}') + pings = [f'payload-{i}' for i in range(0, 10)] - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload) + + ws.pongs_read(sock, pings) # 2_11 - for i in range(0, 10): - opcode = ws.OP_PING - ws.frame_write(sock, opcode, f'payload-{i}', chopsize=1) + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload, chopsize=1) - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + ws.pongs_read(sock, pings) close_connection(sock) diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 1d4dd1184..8dfbd126d 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -495,22 +495,19 @@ def test_node_websockets_2_10__2_11(): _, sock, _ = ws.upgrade() - for i in range(0, 10): - ws.frame_write(sock, ws.OP_PING, f'payload-{i}') + pings = [f'payload-{i}' for i in range(0, 10)] - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload) + + ws.pongs_read(sock, pings) # 2_11 - for i in range(0, 10): - opcode = ws.OP_PING - ws.frame_write(sock, opcode, f'payload-{i}', chopsize=1) + for payload in pings: + ws.frame_write(sock, ws.OP_PING, payload, chopsize=1) - for i in range(0, 10): - frame = ws.frame_read(sock) - check_frame(frame, True, ws.OP_PONG, f'payload-{i}') + ws.pongs_read(sock, pings) close_connection(sock) diff --git a/test/unit/applications/websockets.py b/test/unit/applications/websockets.py index 0125c4e9e..4922bbfb8 100644 --- a/test/unit/applications/websockets.py +++ b/test/unit/applications/websockets.py @@ -34,7 +34,7 @@ def accept(self, key): sha1 = hashlib.sha1((key + GUID).encode()).digest() return base64.b64encode(sha1).decode() - def upgrade(self, headers=None): + def upgrade(self, headers=None, sock=None): key = None if headers is None: @@ -48,10 +48,9 @@ def upgrade(self, headers=None): 'Sec-WebSocket-Version': 13, } - sock = self.get( - headers=headers, - no_recv=True, - ) + kwargs = {} if sock is None else {'sock': sock} + + sock = self.get(headers=headers, no_recv=True, **kwargs) resp = '' while True: @@ -144,6 +143,27 @@ def recv_bytes(sock, bytes_len): return frame + def pongs_read(self, sock, pings, read_timeout=60): + # RFC 6455 Section 5.5.3: while a PONG is not sent, the server can + # answer only the most recent PING. Read PONGs until the one for the + # last PING. They must answer PINGs in the order they were sent. + + pings = [p.encode() if isinstance(p, str) else p for p in pings] + answered = [] + + while not answered or answered[-1] != len(pings) - 1: + frame = self.frame_read(sock, read_timeout) + + assert frame['opcode'] == self.OP_PONG, 'pong opcode' + assert frame['data'] in pings, 'pong payload' + + i = pings.index(frame['data']) + assert not answered or i > answered[-1], 'pong order' + + answered.append(i) + + return answered + def frame_to_send( self, opcode, From 387b02cb8c03a2f5cf75060f63d1613a58bb207a Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 3 Oct 2026 21:19:45 +0200 Subject: [PATCH 353/391] lib: lower the CPU count to the cgroup v2 CPU limit nxt_ncpu is the number of CPUs in the affinity mask. A CPU bandwidth limit (cgroup v2 cpu.max) does not change that mask. A Kubernetes CPU limit is such a limit, so a pod limited to 10 CPUs on a 64-CPU node started 64 router threads, the default of settings/listen_threads. Now nxt_lib_start() reads cpu.max in the cgroup of the process and in each parent cgroup up to the root of the cgroup2 mount. The lowest limit, rounded up to whole CPUs, lowers nxt_ncpu. It never raises it. Go 1.25 and OpenJDK use the same rounding. "max" means no limit. The root of the cgroup2 mount is the root of the cgroup namespace of unitd, not always the root of the host. With a private cgroup namespace, which Docker uses by default on cgroup v2, unitd sees only the limits inside the namespace. A limit on a parent cgroup, such as a systemd slice or a pod-level limit, is not seen. A container limit is set on the cgroup of the container, so it is seen. An explicit settings/listen_threads value still wins. The limit is read once, at startup. After the limit is changed at runtime, unitd must be restarted to use the new limit. cgroup v1 is not read. cpu.max is parsed through nxt_span_t. The mount root and the mount point from /proc/self/mountinfo are decoded from their octal escapes, such as "\040" for a space. A cgroup path with a ".." component is outside the namespace and is skipped. A cgroup named "..foo" is not skipped. nxt_ncpu also sets the spinlock spin count, so a limit of 1 CPU now turns spinning off, as an affinity mask of 1 CPU does. Closes https://github.com/freeunitorg/freeunit/issues/547 Reviewed-by: Claude Fable 5.1 Co-Authored-By: Claude Opus 5.5 --- CHANGES | 10 + auto/sources | 1 + docs/changes.xml | 13 + src/nxt_lib.c | 358 ++++++++++++++++++++++++++ src/nxt_main.h | 4 + src/test/nxt_cpu_limit_test.c | 129 ++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/syscalls/baseline-linux-musl.txt | 1 + test/test_app_lifecycle.py | 14 +- 10 files changed, 529 insertions(+), 6 deletions(-) create mode 100644 src/test/nxt_cpu_limit_test.c diff --git a/CHANGES b/CHANGES index d6ca94ea7..fe0d84d61 100644 --- a/CHANGES +++ b/CHANGES @@ -138,6 +138,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 close a descriptor that another connection had got in the meantime. + *) Change: on Linux, the default number of router threads + ("listen_threads") is now at most the CPU limit of the cgroup v2 + group of unitd and of its parent groups ("cpu.max"), rounded up to + whole CPUs. Before, it was the number of CPUs that unitd could run + on, so a container limited to 2 CPUs on a 64-CPU host started 64 + router threads. Now it starts 2. A limit of 1.5 CPUs also gives 2 + threads. An explicit "listen_threads" is not changed. With a private + cgroup namespace, only the limits inside the namespace are seen. The + limit is read once, when unitd starts. cgroup v1 is not read. + *) Bugfix: a proxied request with a chunked body, or an HTTP/2 request without a content-length, was sent to the upstream with two Content-Length fields. diff --git a/auto/sources b/auto/sources index aa7e92c64..8e417aa7b 100644 --- a/auto/sources +++ b/auto/sources @@ -225,6 +225,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_port_frag_test.c \ src/test/nxt_port_release_test.c \ src/test/nxt_nncq_bound_test.c \ + src/test/nxt_cpu_limit_test.c \ " diff --git a/docs/changes.xml b/docs/changes.xml index 2ba0001c8..de4d61c28 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,19 @@ date="2026-08-30" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +on Linux, the default number of router threads ("listen_threads") is now at +most the CPU limit of the cgroup v2 group of unitd and of its parent groups +("cpu.max"), rounded up to whole CPUs. Before, it was the number of CPUs that +unitd could run on, so a container limited to 2 CPUs on a 64-CPU host started +64 router threads. Now it starts 2. A limit of 1.5 CPUs also gives 2 +threads. An explicit "listen_threads" is not changed. With a private cgroup +namespace, only the limits inside the namespace are seen. The limit is read +once, when unitd starts. cgroup v1 is not read. + + + after 2^32 RPC registrations the stream counter wrapped and handed out diff --git a/src/nxt_lib.c b/src/nxt_lib.c index aaec1e8ea..c2bcaa46a 100644 --- a/src/nxt_lib.c +++ b/src/nxt_lib.c @@ -5,6 +5,7 @@ */ #include +#include nxt_uint_t nxt_ncpu = 1; @@ -15,6 +16,12 @@ nxt_atomic_t nxt_task_ident; nxt_thread_declare_data(nxt_thread_t, nxt_thread_context); +#if (NXT_LINUX) +static nxt_uint_t nxt_cgroup_cpu_limit(void); +static nxt_uint_t nxt_cgroup_cpu_max_read(const char *dir); +#endif + + #if (NXT_DEBUG && NXT_FREEBSD) /* * Fill memory with 0xA5 after malloc() and with 0x5A before free(). @@ -36,6 +43,9 @@ nxt_lib_start(const char *app, char **argv, char ***envp) nxt_int_t flags; nxt_bool_t update; nxt_thread_t *thread; +#if (NXT_LINUX) + nxt_uint_t limit; +#endif flags = nxt_stderr_start(); @@ -115,6 +125,25 @@ nxt_lib_start(const char *app, char **argv, char ***envp) #endif +#if (NXT_LINUX) + /* + * A CPU bandwidth limit (cgroup v2 "cpu.max") does not change the + * affinity mask. A container limited to 2 CPUs on a 64-CPU host + * otherwise starts 64 router threads. + */ + if (n > 1) { + limit = nxt_cgroup_cpu_limit(); + + if (limit != 0 && limit < (nxt_uint_t) n) { + nxt_log(&nxt_main_task, NXT_LOG_INFO, + "the cgroup CPU limit lowers the CPU count from %d to %ui", + n, limit); + + n = (int) limit; + } + } +#endif + nxt_debug(&nxt_main_task, "ncpu: %d", n); if (n > 1) { @@ -143,6 +172,335 @@ nxt_lib_start(const char *app, char **argv, char ***envp) } +/* + * Parses the contents of a cgroup v2 "cpu.max" file: " ", + * or "max " for no limit. Returns the quota in whole CPUs, rounded + * up, or 0 if there is no limit or the contents are not valid. + */ + +nxt_uint_t +nxt_cgroup_cpu_max_parse(const u_char *p, size_t len) +{ + size_t n; + nxt_off_t quota, period; + nxt_span_t span; + const u_char *sp, *field; + + /* A trailing newline or space is not part of the value. */ + + while (len != 0 && (p[len - 1] == '\n' || p[len - 1] == ' ')) { + len--; + } + + nxt_span_init(&span, p, p + len); + + /* The quota is the bytes before the first space. */ + + sp = memchr(p, ' ', len); + if (sp == NULL) { + return 0; + } + + n = sp - p; + + if (nxt_span_take(&span, n, &field) != 0) { + return 0; + } + + quota = nxt_off_t_parse(field, n); + + /* The space. */ + + if (nxt_span_take(&span, 1, &field) != 0) { + return 0; + } + + /* The period is the rest. */ + + n = nxt_span_len(&span); + + if (nxt_span_take(&span, n, &field) != 0) { + return 0; + } + + period = nxt_off_t_parse(field, n); + + if (quota <= 0 || period <= 0) { + return 0; + } + + /* Rounds up without an addition that could overflow. */ + quota = quota / period + (quota % period != 0); + + return (quota > (nxt_off_t) NXT_INT32_T_MAX) ? 0 : (nxt_uint_t) quota; +} + + +/* + * /proc/self/mountinfo writes a space, a tab, a newline and a backslash in + * a path as a backslash and three octal digits, such as "\040" for a space. + * Decodes these escapes in place. /proc/self/cgroup does not escape a + * path, so the decoded mount root can be compared with it. + */ + +void +nxt_cgroup_mountinfo_unescape(char *s) +{ + char *d; + + d = s; + + while (*s != '\0') { + + if (s[0] == '\\' + && s[1] >= '0' && s[1] <= '3' + && s[2] >= '0' && s[2] <= '7' + && s[3] >= '0' && s[3] <= '7') + { + *d++ = (char) (((s[1] - '0') << 6) | ((s[2] - '0') << 3) + | (s[3] - '0')); + s += 4; + continue; + } + + *d++ = *s++; + } + + *d = '\0'; +} + + +/* + * Returns the path of "cgroup" below the root "root" of the cgroup2 mount: + * "" for the root itself, else a path that starts with "/". Returns NULL + * if the cgroup is not below the root, or if its path has a ".." component. + * Such a cgroup is outside the cgroup namespace, and the mount does not + * show it. + */ + +const char * +nxt_cgroup_relative_path(const char *cgroup, const char *root) +{ + size_t len; + const char *rel, *p; + + rel = cgroup; + + if (strcmp(root, "/") != 0) { + len = strlen(root); + + if (strncmp(cgroup, root, len) != 0 + || (cgroup[len] != '/' && cgroup[len] != '\0')) + { + return NULL; + } + + rel = cgroup + len; + } + + /* Only a whole ".." component: a cgroup can be named "..foo". */ + + for (p = strstr(rel, "/.."); p != NULL; p = strstr(p + 1, "/..")) { + + if (p[3] == '/' || p[3] == '\0') { + return NULL; + } + } + + if (strcmp(rel, "/") == 0) { + return ""; + } + + return rel; +} + + +#if (NXT_LINUX) + +/* + * Returns the lowest CPU limit of the cgroup v2 directories from the + * cgroup of this process up to the root of the cgroup2 mount, or 0 if + * no level sets a limit. The root of the mount is the root of the cgroup + * namespace of the process. With a private cgroup namespace, as in a + * container, a limit on a parent of the namespace root is not seen. The + * cgroup v1 "cpu" controller is not read. + */ + +static nxt_uint_t +nxt_cgroup_cpu_limit(void) +{ + int i, len; + char *line, *p, *cgroup, *root, *mnt, *slash; + FILE *fp; + size_t size, base; + ssize_t nread; + nxt_uint_t limit, cpus; + const char *rel; + char dir[NXT_MAX_PATH_LEN]; + + line = NULL; + size = 0; + cgroup = NULL; + limit = 0; + + fp = fopen("/proc/self/cgroup", "re"); + if (fp == NULL) { + return 0; + } + + while ((nread = getline(&line, &size, fp)) > 0) { + if (nread > 3 && memcmp(line, "0::", 3) == 0) { + if (line[nread - 1] == '\n') { + line[nread - 1] = '\0'; + } + + cgroup = strdup(line + 3); + break; + } + } + + fclose(fp); + + if (cgroup == NULL) { + goto done; + } + + fp = fopen("/proc/self/mountinfo", "re"); + if (fp == NULL) { + goto done; + } + + root = NULL; + mnt = NULL; + + while (getline(&line, &size, fp) > 0) { + p = strstr(line, " - cgroup2 "); + if (p == NULL) { + continue; + } + + *p = '\0'; + + /* " ..." */ + + p = line; + + for (i = 0; i < 3 && p != NULL; i++) { + p = strchr(p, ' '); + + if (p != NULL) { + p++; + } + } + + if (p == NULL) { + continue; + } + + root = p; + + p = strchr(p, ' '); + if (p == NULL) { + continue; + } + + *p++ = '\0'; + mnt = p; + + p = strchr(p, ' '); + if (p != NULL) { + *p = '\0'; + } + + break; + } + + fclose(fp); + + if (mnt == NULL) { + goto done; + } + + nxt_cgroup_mountinfo_unescape(root); + nxt_cgroup_mountinfo_unescape(mnt); + + /* The cgroup path is relative to the root of the cgroup namespace. */ + + rel = nxt_cgroup_relative_path(cgroup, root); + if (rel == NULL) { + goto done; + } + + len = snprintf(dir, sizeof(dir), "%s%s", mnt, rel); + if (len < 0 || (size_t) len >= sizeof(dir)) { + goto done; + } + + base = strlen(mnt); + + for ( ;; ) { + cpus = nxt_cgroup_cpu_max_read(dir); + + if (cpus != 0 && (limit == 0 || cpus < limit)) { + limit = cpus; + } + + if ((size_t) len <= base) { + break; + } + + slash = strrchr(dir + base, '/'); + if (slash == NULL) { + break; + } + + *slash = '\0'; + len = slash - dir; + } + +done: + + nxt_free(cgroup); + nxt_free(line); + + return limit; +} + + +static nxt_uint_t +nxt_cgroup_cpu_max_read(const char *dir) +{ + int fd, len; + ssize_t n; + u_char buf[64]; + char path[NXT_MAX_PATH_LEN]; + + len = snprintf(path, sizeof(path), "%s/cpu.max", dir); + if (len < 0 || (size_t) len >= sizeof(path)) { + return 0; + } + + /* The root cgroup and a cgroup without the "cpu" controller have none. */ + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd == -1) { + return 0; + } + + n = read(fd, buf, sizeof(buf)); + + close(fd); + + if (n <= 0) { + return 0; + } + + return nxt_cgroup_cpu_max_parse(buf, n); +} + +#endif + + void nxt_lib_stop(void) { diff --git a/src/nxt_main.h b/src/nxt_main.h index ab10e6848..ba4057388 100644 --- a/src/nxt_main.h +++ b/src/nxt_main.h @@ -157,6 +157,10 @@ typedef void (*nxt_event_conn_handler_t)(nxt_thread_t *thr, nxt_conn_t *c); */ NXT_EXPORT nxt_int_t nxt_lib_start(const char *app, char **argv, char ***envp); NXT_EXPORT void nxt_lib_stop(void); +NXT_EXPORT nxt_uint_t nxt_cgroup_cpu_max_parse(const u_char *p, size_t len); +NXT_EXPORT void nxt_cgroup_mountinfo_unescape(char *s); +NXT_EXPORT const char *nxt_cgroup_relative_path(const char *cgroup, + const char *root); NXT_EXPORT extern nxt_uint_t nxt_ncpu; diff --git a/src/test/nxt_cpu_limit_test.c b/src/test/nxt_cpu_limit_test.c new file mode 100644 index 000000000..9640c8253 --- /dev/null +++ b/src/test/nxt_cpu_limit_test.c @@ -0,0 +1,129 @@ + +/* + * Copyright (C) FreeUnit + */ + +#include +#include "nxt_tests.h" + + +typedef struct { + const char *in; + nxt_uint_t cpus; +} nxt_cpu_limit_case_t; + + +typedef struct { + const char *in; + const char *out; +} nxt_cpu_limit_str_case_t; + + +typedef struct { + const char *cgroup; + const char *root; + const char *rel; +} nxt_cpu_limit_path_case_t; + + +nxt_int_t +nxt_cpu_limit_test(nxt_thread_t *thr) +{ + nxt_uint_t i, cpus; + nxt_bool_t ok; + const char *rel; + char buf[32]; + + static const nxt_cpu_limit_case_t cases[] = { + { "max 100000\n", 0 }, + { "200000 100000\n", 2 }, + { "150000 100000\n", 2 }, + { "50000 100000\n", 1 }, + { "1000 1000000\n", 1 }, + { "6400000 100000", 64 }, + { "100000 0\n", 0 }, + { "0 100000\n", 0 }, + { "-1 100000\n", 0 }, + { "100000\n", 0 }, + { "100000 \n", 0 }, + { "\n", 0 }, + { "", 0 }, + { "1x0000 100000\n", 0 }, + { "9223372036854775807 1\n", 0 }, + { "100001 100000\n", 2 }, + { "2147483647 1\n", 2147483647 }, + { "2147483648 1\n", 0 }, + { "200000 100000 \n \n", 2 }, + { " 200000 100000\n", 0 }, + { "200000 100000\n", 0 }, + { "200000 100000 1\n", 0 }, + { "200000\n100000\n", 0 }, + { " \n", 0 }, + }; + + static const nxt_cpu_limit_str_case_t unescape[] = { + { "/sys/fs/cgroup", "/sys/fs/cgroup" }, + { "/mnt/a\\040b", "/mnt/a b" }, + { "\\011\\012\\134\\040", "\t\n\\ " }, + { "/a\\04", "/a\\04" }, + { "/a\\0x0", "/a\\0x0" }, + { "/a\\400", "/a\\400" }, + { "/a\\", "/a\\" }, + { "", "" }, + }; + + static const nxt_cpu_limit_path_case_t paths[] = { + { "/", "/", "" }, + { "/a/b", "/", "/a/b" }, + { "/a/b", "/a", "/b" }, + { "/a", "/a", "" }, + { "/ab", "/a", NULL }, + { "/b", "/a", NULL }, + { "/..", "/", NULL }, + { "/../b", "/", NULL }, + { "/a/..", "/", NULL }, + { "/a/../b", "/", NULL }, + { "/..a", "/", "/..a" }, + { "/a/..b/c", "/", "/a/..b/c" }, + { "/a..", "/", "/a.." }, + { "/a/b..c", "/a", "/b..c" }, + }; + + for (i = 0; i < nxt_nitems(cases); i++) { + cpus = nxt_cgroup_cpu_max_parse((const u_char *) cases[i].in, + strlen(cases[i].in)); + + NXT_TEST_CHECK(thr->log, cpus == cases[i].cpus, + "cpu.max \"%s\": %ui CPUs, expected %ui", + cases[i].in, cpus, cases[i].cpus); + } + + for (i = 0; i < nxt_nitems(unescape); i++) { + nxt_cpystrn((u_char *) buf, (const u_char *) unescape[i].in, + sizeof(buf)); + + nxt_cgroup_mountinfo_unescape(buf); + + NXT_TEST_CHECK(thr->log, strcmp(buf, unescape[i].out) == 0, + "mountinfo \"%s\": \"%s\", expected \"%s\"", + unescape[i].in, buf, unescape[i].out); + } + + for (i = 0; i < nxt_nitems(paths); i++) { + rel = nxt_cgroup_relative_path(paths[i].cgroup, paths[i].root); + + ok = (rel == NULL || paths[i].rel == NULL) + ? rel == paths[i].rel + : strcmp(rel, paths[i].rel) == 0; + + NXT_TEST_CHECK(thr->log, ok, + "cgroup \"%s\" below \"%s\": \"%s\", expected \"%s\"", + paths[i].cgroup, paths[i].root, + (rel != NULL) ? rel : "NULL", + (paths[i].rel != NULL) ? paths[i].rel : "NULL"); + } + + nxt_log_error(NXT_LOG_NOTICE, thr->log, "cpu limit test passed"); + + return NXT_OK; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 78e4e0f9a..112790639 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -423,6 +423,10 @@ main(int argc, char **argv) return 1; } + if (nxt_cpu_limit_test(thr) != NXT_OK) { + return 1; + } + #if (NXT_HAVE_CLONE_NEWUSER) if (nxt_clone_creds_test(thr) != NXT_OK) { return 1; diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index ca240ef83..7e3ad9ab8 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -120,6 +120,7 @@ nxt_int_t nxt_regex_test(nxt_thread_t *thr); nxt_int_t nxt_openssl_server_init_test(nxt_thread_t *thr); nxt_int_t nxt_cgroup_test(nxt_thread_t *thr); nxt_int_t nxt_controller_peer_test(nxt_thread_t *thr); +nxt_int_t nxt_cpu_limit_test(nxt_thread_t *thr); nxt_int_t nxt_clone_creds_test(nxt_thread_t *thr); #if (NXT_HAVE_ISOLATION_ROOTFS) nxt_int_t nxt_isolation_mount_dst_test(nxt_thread_t *thr); diff --git a/test/syscalls/baseline-linux-musl.txt b/test/syscalls/baseline-linux-musl.txt index df4716c79..d005c7901 100644 --- a/test/syscalls/baseline-linux-musl.txt +++ b/test/syscalls/baseline-linux-musl.txt @@ -37,6 +37,7 @@ getsockopt gettid ioctl listen +lseek lstat membarrier memfd_create diff --git a/test/test_app_lifecycle.py b/test/test_app_lifecycle.py index a75742dc2..71003fc68 100644 --- a/test/test_app_lifecycle.py +++ b/test/test_app_lifecycle.py @@ -812,10 +812,11 @@ def test_app_lifecycle_app_fds_stable(app): The socket rule is therefore one bound, and the discovery ceiling is what makes it safe. There are at most nxt_ncpu engines, the test does - not touch `listen_threads`, and nxt_ncpu is CPU_COUNT() of the affinity - mask, capped by _SC_NPROCESSORS_ONLN (nxt_lib.c) -- which is what - os.sched_getaffinity() reports here. So: total socket growth across the - two windows may not exceed that ceiling. Discovery can never exceed it, + not touch `listen_threads`, and nxt_ncpu is at most CPU_COUNT() of the + affinity mask, capped by _SC_NPROCESSORS_ONLN (nxt_lib.c) -- which is + what os.sched_getaffinity() reports here. A cgroup v2 CPU limit can + only lower nxt_ncpu. So: total socket growth across the two windows may + not exceed that ceiling. Discovery can never exceed it, however the first touches fall across the windows -- one window, both, or a whole burst's worth of fresh engines on a machine with more CPUs than the burst has requests -- so this bound cannot be tripped by the @@ -961,8 +962,9 @@ def burst(pool): ) # sockets: bounded by the engine count, because port discovery is lazy - # and one-time per engine. nxt_ncpu is CPU_COUNT() of the affinity mask - # (src/nxt_lib.c) and the test does not touch `listen_threads`. + # and one-time per engine. nxt_ncpu is at most CPU_COUNT() of the + # affinity mask; a cgroup v2 CPU limit can only lower it (src/nxt_lib.c). + # The test does not touch `listen_threads`. ncpu = max(len(os.sched_getaffinity(0)), 1) sockets = [n.sockets for n in counts] From c6518d86633a6cbf1b4072016658bf0f83f682db Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 07:55:10 +0200 Subject: [PATCH 354/391] php: free the root path buffer on every error path of set_target nxt_php_set_target() returned NXT_ERROR without freeing the root path buffer when realpath() failed. After the root was set, the script and index branches also returned without freeing it. All these paths now free the buffer and return through one label. No caller frees the target on this error, so there is no double free. After it frees them, the function now sets target->root.start to NULL, and target->script_filename.start to NULL on the "script is not under php root" and nxt_php_dirname() failures. nxt_php_cleanup_targets() frees every field that is not NULL. A later cleanup on the error path of nxt_php_start() must not free these buffers a second time. The function also sets the length of each cleared field to 0, and does the same for script_dirname, which nxt_php_dirname() can leave with a length and a NULL start. nxt_php_request_handler() decides by script_filename.length, so a field with a length and a NULL start would reach fopen() and chdir() as NULL. No test covers this. Every error return of nxt_php_set_target() ends with the worker exiting, so no request can reach the handler. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Claude Sonnet 5.5 Reviewed-by: DeepSeek V4.1 Flash --- src/nxt_php_sapi.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index f9a823fc7..9ef5b5af5 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -577,6 +577,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, p = nxt_realpath(tmp); if (nxt_slow_path(p == NULL)) { nxt_alert(task, "root realpath(%s) failed %E", tmp, nxt_errno); + nxt_free(tmp); return NXT_ERROR; } @@ -596,7 +597,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, tmp = nxt_malloc(target->root.length + 1 + str.length + 1); if (nxt_slow_path(tmp == NULL)) { - return NXT_ERROR; + goto fail; } p = tmp; @@ -611,7 +612,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, if (nxt_slow_path(p == NULL)) { nxt_alert(task, "script realpath(%s) failed %E", tmp, nxt_errno); nxt_free(tmp); - return NXT_ERROR; + goto fail; } nxt_free(tmp); @@ -624,14 +625,17 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, { nxt_alert(task, "script is not under php root"); nxt_free(p); - return NXT_ERROR; + nxt_str_null(&target->script_filename); + goto fail; } ret = nxt_php_dirname(&target->script_filename, &target->script_dirname); if (nxt_slow_path(ret != NXT_OK)) { nxt_free(target->script_filename.start); - return NXT_ERROR; + nxt_str_null(&target->script_filename); + nxt_str_null(&target->script_dirname); + goto fail; } target->script_name.length = target->script_filename.length @@ -647,7 +651,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, tmp = nxt_malloc(str.length + 1); if (nxt_slow_path(tmp == NULL)) { - return NXT_ERROR; + goto fail; } nxt_memcpy(tmp, str.start, str.length); @@ -663,6 +667,13 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, } return NXT_OK; + +fail: + + nxt_free(target->root.start); + nxt_str_null(&target->root); + + return NXT_ERROR; } From 4a6aad0593ecce7e965e0974e38e16312bb39eb0 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 30 Sep 2026 03:39:28 +0200 Subject: [PATCH 355/391] main: at exit, kill a state store child that a newer store replaces Since commit b12fb67788838f9ffe829df98adbe4c7a523cf6d, a store child runs the fsync(2) calls of a conf.json store. A store that arrives while a child runs becomes the pending store, and it starts when the child exits. At exit, main waits for the child and then for the pending store. After PUT, PUT, SIGTERM it waited for two stores in sequence, although the first one stored a configuration that was no longer the last one. Now, when main starts to exit and a store is pending, it sends SIGKILL to the running store child. A store that becomes pending during the exit does the same. The pending store starts when the SIGCHLD handler reaps the child, as before, so two stores still do not overlap. PUT, PUT, SIGTERM now waits for one store and the rest of the killed one: SIGKILL takes effect when the fsync(2) in progress returns. nxt_main_store_cancel() does the kill. nxt_main_start_exit() sets nxt_exiting and calls it. The SIGTERM and SIGQUIT handlers call nxt_main_start_exit(), and so does the start failure. See https://github.com/freeunitorg/freeunit/issues/516. While main runs, nothing changes: the child finishes its store, and the pending store waits for it. A kill on each new store would starve the store. A review of an earlier version of this change, which killed on each new store, measured it: with fsync(2) delayed to 22 ms by "strace -f --seccomp-bpf -e inject=fsync:delay_exit=22000", conf.json did not change during a 4 s loop of PUTs. With this version conf.json stays as close to the last PUT as on master. On pro4s-2, the same injection, a 4 s PUT loop, conf.json read every 50 ms, two runs each: build PUTs longest unchanged max stale kill on each new store 6240 4000 ms 3964 ms 5564 4000 ms 3964 ms no kill (as on master) 5595 100 ms 102 ms 6125 100 ms 102 ms this commit (--debug) 1024 100 ms 113 ms 1000 100 ms 104 ms "max stale" is the longest time conf.json held a value while a newer PUT had already been answered. The last value was stored in every run. A kill does not lose a store: - A killed child leaves at most the temporary file. The next store unlinks it first. - A child killed after a rename(2) and before the directory fsync has put a complete file in place, because it flushed the file before the rename. The next store flushes the directory again. - A killed child does not set nxt_conf_ver, so the next store writes the version file again. - A child that exited before the signal arrived keeps its exit code. kill(2) of a zombie succeeds and changes nothing. Main does not log a child that it killed as a failed store. The SIGCHLD handler logs it with the "process N exited on signal 9" line at the notice level, not as an alert. Tests: - src/test/nxt_main_store_test.c drives the scheduler. An NXT_TESTS hook makes each store child sleep before it stores. Another hook calls nxt_main_start_exit(), the function that the signal handlers call, so the test runs their code. Without the nxt_main_store_cancel() call in nxt_main_start_exit() the test fails: "the superseded store child N was not killed at exit: it exited with status 0x0". While main runs, a second store must not kill the first child: it must exit with 0 and store its configuration. At exit, the running child must die of SIGKILL, both when the exit starts with a store pending and when a store comes during the exit, with no alert and with nxt_conf_ver unset; the last store then stores the configuration and the version file. A child that exited before the kill counts as a successful store. - test/test_state_store.py: the ENOSPC case now waits for the alert of main. Before, skip_alert() only allowed it, and nothing checked that it was logged. CHANGES and docs/changes.xml: the unreleased entry of commit b12fb67788838f9ffe829df98adbe4c7a523cf6d now says that at exit main stops a running store when a newer configuration waits. test/syscalls: main now can call kill(2), so both baselines list it. The capture sees it only when SIGQUIT arrives while a store child runs and a newer store waits. Without the entry, the gate would fail at random. capture.sh now waits for conf.json before its second PUT. A kill before the first child created conf.json would leave the second store without a destination, and the capture would lose the fchmod(2) and fchown(2) that the second PUT is there to cover. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 5 +- auto/sources | 1 + docs/changes.xml | 3 +- src/nxt_main_process.c | 157 ++++++- src/nxt_main_process.h | 6 + src/test/nxt_main_store_test.c | 609 +++++++++++++++++++++++++ src/test/nxt_tests.c | 4 + src/test/nxt_tests.h | 1 + test/syscalls/baseline-linux-glibc.txt | 1 + test/syscalls/baseline-linux-musl.txt | 1 + test/syscalls/capture.sh | 11 + test/test_state_store.py | 9 +- 12 files changed, 797 insertions(+), 11 deletions(-) create mode 100644 src/test/nxt_main_store_test.c diff --git a/CHANGES b/CHANGES index fe0d84d61..0f7baf2b6 100644 --- a/CHANGES +++ b/CHANGES @@ -62,8 +62,9 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 the start of application processes after a reconfiguration, or the exit of the main process. One store runs at a time; a newer configuration waits for it, and only the latest one waiting is - stored. At exit, the main process waits for the store to finish. - When the store fails, the main process logs an alert. + stored. At exit, the main process waits for the store to finish; + when a newer configuration waits, it stops the running store + first. When the store fails, the main process logs an alert. *) Bugfix: a regular expression in the configuration could run for up to 10,000,000 match steps on one request. A match now stops after diff --git a/auto/sources b/auto/sources index 8e417aa7b..d94bacdb6 100644 --- a/auto/sources +++ b/auto/sources @@ -209,6 +209,7 @@ NXT_TEST_SRCS=" \ src/test/nxt_router_websocket_test.c \ src/test/nxt_main_start_process_reply_test.c \ src/test/nxt_main_file_store_test.c \ + src/test/nxt_main_store_test.c \ src/test/nxt_proto_creating_wedge_test.c \ src/test/nxt_main_whoami_test.c \ src/test/nxt_main_remove_child_pid_test.c \ diff --git a/docs/changes.xml b/docs/changes.xml index de4d61c28..2fa669996 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -111,7 +111,8 @@ process, so a slow fsync(2) no longer delays the start of application processes after a reconfiguration, or the exit of the main process. One store runs at a time; a newer configuration waits for it, and only the latest one waiting is stored. At exit, the main process waits for the -store to finish. When the store fails, the main process logs an alert. +store to finish; when a newer configuration waits, it stops the running +store first. When the store fails, the main process logs an alert. diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 51aed83c1..6fd89ce03 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -39,6 +39,7 @@ typedef struct { typedef struct { nxt_pid_t pid; /* The running store child, or 0. */ nxt_bool_t version; /* It also stores the version file. */ + nxt_bool_t killed; /* Main sent it SIGKILL. */ u_char *pending; /* The next store's mapping, or NULL. */ size_t pending_size; } nxt_main_store_t; @@ -82,6 +83,9 @@ static void nxt_main_store_schedule(nxt_task_t *task, u_char *p, static void nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size); static nxt_int_t nxt_main_store_files(nxt_task_t *task, u_char *p, size_t size, nxt_bool_t version); +static void nxt_main_start_exit(nxt_task_t *task); +static void nxt_main_store_cancel(nxt_task_t *task); +static nxt_bool_t nxt_main_store_cancelled(nxt_pid_t pid, int status); static nxt_bool_t nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, int status); static nxt_int_t nxt_main_file_store_inherit(nxt_task_t *task, @@ -91,6 +95,7 @@ static void nxt_main_port_access_log_handler(nxt_task_t *task, #if (NXT_TESTS) static nxt_uint_t nxt_main_test_process_new_failure_count; +static nxt_msec_t nxt_main_test_store_delay; void @@ -775,6 +780,53 @@ nxt_main_test_run_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Let src/test/nxt_main_store_test.c drive the state store. The delay + * makes each store child sleep before it stores, so that the test can act + * while the child runs. The test reaps the child with waitpid() and + * gives the status to nxt_main_test_store_exited(). + */ +void +nxt_main_test_store_set_delay(nxt_msec_t delay) +{ + nxt_main_test_store_delay = delay; +} + + +void +nxt_main_test_store_schedule(nxt_task_t *task, u_char *p, size_t size) +{ + nxt_main_store_schedule(task, p, size); +} + + +nxt_pid_t +nxt_main_test_store_pid(void) +{ + return nxt_main_store.pid; +} + + +nxt_bool_t +nxt_main_test_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) +{ + return nxt_main_store_exited(task, pid, status); +} + + +/* Start the exit as the SIGTERM and SIGQUIT handlers do, or end it. */ +void +nxt_main_test_store_set_exiting(nxt_task_t *task, nxt_bool_t exiting) +{ + if (exiting) { + nxt_main_start_exit(task); + + } else { + nxt_exiting = 0; + } +} + + #if (NXT_USE_CMSG_PID) void @@ -1268,7 +1320,7 @@ nxt_main_process_sigterm_handler(nxt_task_t *task, void *obj, void *data) */ rt->quit_mode = NXT_PORT_QUIT_NORMAL; - nxt_exiting = 1; + nxt_main_start_exit(task); nxt_runtime_quit(task, 0); } @@ -1290,7 +1342,7 @@ nxt_main_process_sigquit_handler(nxt_task_t *task, void *obj, void *data) */ rt->quit_mode = NXT_PORT_QUIT_GRACEFUL; - nxt_exiting = 1; + nxt_main_start_exit(task); nxt_runtime_quit(task, 0); } @@ -1437,7 +1489,12 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) return; } - if (WTERMSIG(status)) { + if (nxt_main_store_cancelled(pid, status)) { + /* Main stopped it; see nxt_main_store_cancel(). */ + nxt_trace(task, "process %PI exited on signal %d", + pid, WTERMSIG(status)); + + } else if (WTERMSIG(status)) { #ifdef WCOREDUMP nxt_alert(task, "process %PI exited on signal %d%s", pid, WTERMSIG(status), @@ -2089,7 +2146,7 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } if (nxt_slow_path(ret == NXT_ERROR)) { - nxt_exiting = 1; + nxt_main_start_exit(task); nxt_runtime_quit(task, 1); } @@ -2228,6 +2285,68 @@ nxt_main_store_schedule(nxt_task_t *task, u_char *p, size_t size) nxt_main_store.pending = p; nxt_main_store.pending_size = size; + + if (nxt_exiting) { + nxt_main_store_cancel(task); + } +} + + +/* + * Main starts to exit. The SIGTERM and SIGQUIT handlers call this, and so + * does the start failure. The test hook nxt_main_test_store_set_exiting() + * calls it too, so src/test/nxt_main_store_test.c runs this code. + */ +static void +nxt_main_start_exit(nxt_task_t *task) +{ + nxt_exiting = 1; + + nxt_main_store_cancel(task); +} + + +/* + * At exit, main waits for the store child and then for the pending store. + * When a store is pending, the running child stores a configuration that + * is no longer the last one, so main stops it with SIGKILL and waits for + * one store instead of two. The pending store starts when the child is + * reaped, so two stores still do not overlap. + * + * Main does this only at exit. While main runs, the child finishes, and + * the pending store waits for it. A kill on each new store would let + * stores that come faster than one store takes keep conf.json old until + * they stop. + * + * A killed child leaves at most the temporary file, and the next store + * unlinks it first. If the child is killed after a rename(), the file it + * renamed is complete, because it was flushed before the rename(); the + * next store flushes the directory again. A killed child does not set + * nxt_conf_ver, so the next store writes the version file again. + */ +static void +nxt_main_store_cancel(nxt_task_t *task) +{ + if (nxt_main_store.pid == 0 + || nxt_main_store.killed + || nxt_main_store.pending == NULL) + { + return; + } + + /* + * The child is not reaped until the SIGCHLD handler runs, so its pid + * cannot belong to another process yet. + */ + if (nxt_slow_path(kill(nxt_main_store.pid, SIGKILL) != 0)) { + nxt_alert(task, "kill(%PI, SIGKILL) failed %E", + nxt_main_store.pid, nxt_errno); + return; + } + + nxt_debug(task, "state store child %PI cancelled", nxt_main_store.pid); + + nxt_main_store.killed = 1; } @@ -2268,6 +2387,12 @@ nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) nxt_pid = getpid(); task->thread->tid = 0; +#if (NXT_TESTS) + if (nxt_main_test_store_delay != 0) { + nxt_nanosleep(nxt_main_test_store_delay * 1000000); + } +#endif + if (nxt_main_store_files(task, p, size, version) != NXT_OK) { _exit(1); } @@ -2319,13 +2444,26 @@ nxt_main_store_files(nxt_task_t *task, u_char *p, size_t size, } +/* Whether "pid" is the store child that main killed, and it died of that. */ +static nxt_bool_t +nxt_main_store_cancelled(nxt_pid_t pid, int status) +{ + return (pid == nxt_main_store.pid + && nxt_main_store.killed + && WIFSIGNALED(status) + && WTERMSIG(status) == SIGKILL); +} + + /* * Called by the SIGCHLD handler for every reaped pid. Returns 1 when "pid" * was the store child. The child is not in the process registry and is not * counted in rt->nprocesses. * * Main logs a failed store again: the alerts of the child can go to a - * rotated log, because SIGUSR1 reopens the log in main only. + * rotated log, because SIGUSR1 reopens the log in main only. A child that + * main killed is not a failure: a newer store is pending. A child that + * exited before the signal arrived keeps its exit code. */ static nxt_bool_t nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) @@ -2337,18 +2475,23 @@ nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) return 0; } - nxt_main_store.pid = 0; - if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { if (nxt_main_store.version) { nxt_conf_ver = NXT_VERNUM; } + } else if (nxt_main_store_cancelled(pid, status)) { + nxt_debug(task, "state store child %PI stopped, a newer store follows", + pid); + } else { nxt_alert(task, "state store child %PI failed, " "the configuration was not stored", pid); } + nxt_main_store.pid = 0; + nxt_main_store.killed = 0; + p = nxt_main_store.pending; if (p != NULL) { diff --git a/src/nxt_main_process.h b/src/nxt_main_process.h index 09ac6cf4a..9c1af277a 100644 --- a/src/nxt_main_process.h +++ b/src/nxt_main_process.h @@ -41,6 +41,12 @@ void nxt_main_test_run_start_process_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_main_test_run_whoami_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +void nxt_main_test_store_set_delay(nxt_msec_t delay); +void nxt_main_test_store_schedule(nxt_task_t *task, u_char *p, size_t size); +nxt_pid_t nxt_main_test_store_pid(void); +nxt_bool_t nxt_main_test_store_exited(nxt_task_t *task, nxt_pid_t pid, + int status); +void nxt_main_test_store_set_exiting(nxt_task_t *task, nxt_bool_t exiting); #if (NXT_USE_CMSG_PID) void nxt_main_test_run_name_child(nxt_task_t *task, nxt_process_t *pprocess, nxt_process_t *process, nxt_pid_t ns_pid); diff --git a/src/test/nxt_main_store_test.c b/src/test/nxt_main_store_test.c new file mode 100644 index 000000000..b80a209cf --- /dev/null +++ b/src/test/nxt_main_store_test.c @@ -0,0 +1,609 @@ +/* + * Copyright (C) F5, Inc. + */ + +/* + * Tests of the state store child of main (src/nxt_main_process.c, + * https://github.com/freeunitorg/freeunit/issues/516). + * + * The test drives the scheduler directly. An NXT_TESTS hook makes each + * store child sleep before it stores, so the test can act while a child + * runs. The test reaps each child with waitpid() and gives the status to + * the handler that the SIGCHLD handler of main calls. + * + * Cases: + * + * - While main runs, a newer store waits for the running store child, + * which finishes its store. + * - At exit, main stops a running store child with SIGKILL when a newer + * store waits: when the exit starts, and when a store comes during + * the exit. The killed child is not logged as a failed store, it does + * not set nxt_conf_ver, and the newer store runs when the child is + * reaped. + * - A child that exited before the signal arrived keeps its exit code: + * its store counts, and nxt_conf_ver is set. + */ + +#include +#include +#include +#include "nxt_tests.h" + +#include +#include + + +#define NXT_MAIN_STORE_TEST_DELAY 500 + + +#define nxt_main_store_test_fail(thr, ...) \ + do { \ + nxt_log_alert((thr)->log, "main store test: " __VA_ARGS__); \ + goto done; \ + } while (0) + + +typedef struct { + char dir[NXT_MAX_PATH_LEN]; + char conf[NXT_MAX_PATH_LEN]; + char conf_tmp[NXT_MAX_PATH_LEN]; + char ver[NXT_MAX_PATH_LEN]; + char ver_tmp[NXT_MAX_PATH_LEN]; +} nxt_main_store_test_paths_t; + + +static nxt_uint_t nxt_main_store_test_alerts; +static nxt_log_handler_t nxt_main_store_test_next_handler; + + +/* Counts the alerts of this process, and logs as before. */ +static void nxt_cdecl +nxt_main_store_test_log(nxt_uint_t level, nxt_log_t *log, const char *fmt, + ...) +{ + u_char *p; + va_list args; + u_char msg[NXT_MAX_ERROR_STR]; + + if (level == NXT_LOG_ALERT) { + nxt_main_store_test_alerts++; + } + + va_start(args, fmt); + p = nxt_vsprintf(msg, msg + sizeof(msg), fmt, args); + va_end(args); + + nxt_main_store_test_next_handler(level, log, "%*s", (size_t) (p - msg), + msg); +} + + +/* The mapping that the scheduler takes and unmaps. */ +static u_char * +nxt_main_store_test_map(const char *content, size_t *size) +{ + u_char *p; + + *size = nxt_strlen(content); + + p = nxt_mem_mmap(NULL, *size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + if (nxt_slow_path(p == MAP_FAILED)) { + return NULL; + } + + nxt_memcpy(p, content, *size); + + return p; +} + + +static nxt_int_t +nxt_main_store_test_schedule(nxt_task_t *task, const char *content) +{ + u_char *p; + size_t size; + + p = nxt_main_store_test_map(content, &size); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + nxt_main_test_store_schedule(task, p, size); + + return NXT_OK; +} + + +static int +nxt_main_store_test_wait(nxt_pid_t pid, int *status) +{ + while (waitpid(pid, status, 0) != pid) { + if (nxt_errno != NXT_EINTR) { + return -1; + } + } + + return 0; +} + + +static nxt_bool_t +nxt_main_store_test_holds(const char *name, const char *expected) +{ + int fd; + char buf[256]; + size_t len; + ssize_t n; + + fd = open(name, O_RDONLY); + if (fd == -1) { + return 0; + } + + n = read(fd, buf, sizeof(buf)); + + close(fd); + + len = nxt_strlen(expected); + + return (n == (ssize_t) len && memcmp(buf, expected, len) == 0); +} + + +/* Stop and reap a store child that a failed case left behind. */ +static void +nxt_main_store_test_reap_all(nxt_task_t *task) +{ + int status; + nxt_pid_t pid; + + nxt_main_test_store_set_delay(0); + nxt_main_test_store_set_exiting(task, 0); + + for ( ;; ) { + pid = nxt_main_test_store_pid(); + if (pid == 0) { + return; + } + + (void) kill(pid, SIGKILL); + + if (nxt_main_store_test_wait(pid, &status) != 0) { + return; + } + + (void) nxt_main_test_store_exited(task, pid, status); + } +} + + +/* + * While main runs, a newer store does not stop the running child. Stores + * that come faster than one store takes would otherwise keep conf.json old + * until they stop. + */ +static nxt_int_t +nxt_main_store_test_running(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + int status; + nxt_int_t ret; + nxt_pid_t first, second; + + ret = NXT_ERROR; + + nxt_main_store_test_alerts = 0; + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + if (nxt_main_store_test_schedule(task, "{\"r\":1}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the first store"); + } + + first = nxt_main_test_store_pid(); + if (first == 0) { + nxt_main_store_test_fail(thr, "the first store started no child"); + } + + if (nxt_main_store_test_schedule(task, "{\"r\":2}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the second store"); + } + + nxt_main_test_store_set_delay(0); + + if (nxt_main_store_test_wait(first, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", first, + nxt_errno); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_main_store_test_fail(thr, "the running store child %PI did not " + "finish while main runs: status 0x%Xi", + first, (nxt_uint_t) status); + } + + (void) nxt_main_test_store_exited(task, first, status); + + if (!nxt_main_store_test_holds(paths->conf, "{\"r\":1}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the first " + "configuration", paths->conf); + } + + second = nxt_main_test_store_pid(); + if (second == 0 || second == first) { + nxt_main_store_test_fail(thr, "the pending store did not start"); + } + + if (nxt_main_store_test_wait(second, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", second, + nxt_errno); + } + + (void) nxt_main_test_store_exited(task, second, status); + + if (!nxt_main_store_test_holds(paths->conf, "{\"r\":2}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the last " + "configuration", paths->conf); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + return ret; +} + + +/* Reap "pid" and require that main killed it, without an alert. */ +static nxt_int_t +nxt_main_store_test_killed(nxt_thread_t *thr, nxt_task_t *task, + nxt_pid_t pid) +{ + int status; + + if (nxt_main_store_test_wait(pid, &status) != 0) { + nxt_log_alert(thr->log, "main store test: waitpid(%PI) failed %E", + pid, nxt_errno); + return NXT_ERROR; + } + + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) { + nxt_log_alert(thr->log, "main store test: the superseded store child " + "%PI was not killed at exit: it exited with status " + "0x%Xi", pid, (nxt_uint_t) status); + return NXT_ERROR; + } + + if (!nxt_main_test_store_exited(task, pid, status)) { + nxt_log_alert(thr->log, "main store test: child %PI is not the store " + "child", pid); + return NXT_ERROR; + } + + if (nxt_main_store_test_alerts != 0) { + nxt_log_alert(thr->log, "main store test: the killed child %PI was " + "logged as a failure (%ui alerts)", pid, + nxt_main_store_test_alerts); + return NXT_ERROR; + } + + if (nxt_conf_ver != 0) { + nxt_log_alert(thr->log, "main store test: a killed child set " + "nxt_conf_ver"); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* + * At exit, a newer store stops the running child: first when the exit + * starts with a store pending, then when a store comes during the exit. + */ +static nxt_int_t +nxt_main_store_test_cancel(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + int status; + char ver[NXT_INT_T_LEN + 1]; + nxt_int_t ret; + nxt_pid_t first, second, third; + struct stat st; + + ret = NXT_ERROR; + + nxt_conf_ver = 0; + nxt_main_store_test_alerts = 0; + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + if (nxt_main_store_test_schedule(task, "{\"a\":1}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the first store"); + } + + first = nxt_main_test_store_pid(); + if (first == 0) { + nxt_main_store_test_fail(thr, "the first store started no child"); + } + + if (nxt_main_store_test_schedule(task, "{\"b\":2}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the second store"); + } + + if (nxt_main_test_store_pid() != first) { + nxt_main_store_test_fail(thr, "a second child started while the " + "first one runs"); + } + + /* SIGTERM or SIGQUIT: the exit starts with a store pending. */ + nxt_main_test_store_set_exiting(task, 1); + + if (nxt_main_store_test_killed(thr, task, first) != NXT_OK) { + goto done; + } + + second = nxt_main_test_store_pid(); + if (second == 0 || second == first) { + nxt_main_store_test_fail(thr, "the pending store did not start when " + "the killed child was reaped"); + } + + /* A store that comes during the exit. */ + + nxt_main_test_store_set_delay(0); + + if (nxt_main_store_test_schedule(task, "{\"c\":3}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the third store"); + } + + if (nxt_main_store_test_killed(thr, task, second) != NXT_OK) { + goto done; + } + + third = nxt_main_test_store_pid(); + if (third == 0 || third == second) { + nxt_main_store_test_fail(thr, "the store that came during the exit " + "did not start"); + } + + if (nxt_main_store_test_wait(third, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", third, + nxt_errno); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_main_store_test_fail(thr, "the last store failed: status " + "0x%Xi", (nxt_uint_t) status); + } + + (void) nxt_main_test_store_exited(task, third, status); + + if (nxt_main_test_store_pid() != 0) { + nxt_main_store_test_fail(thr, "a store child runs with nothing " + "pending"); + } + + if (nxt_conf_ver != NXT_VERNUM) { + nxt_main_store_test_fail(thr, "nxt_conf_ver is not set after the " + "version was stored"); + } + + if (!nxt_main_store_test_holds(paths->conf, "{\"c\":3}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the last " + "configuration", paths->conf); + } + + (void) nxt_sprintf((u_char *) ver, (u_char *) ver + sizeof(ver), "%d%Z", + NXT_VERNUM); + + if (!nxt_main_store_test_holds(paths->ver, ver)) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the version", + paths->ver); + } + + if (stat(paths->conf_tmp, &st) == 0 || stat(paths->ver_tmp, &st) == 0) { + nxt_main_store_test_fail(thr, "a temporary file was left behind"); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + return ret; +} + + +/* + * The child finished its store before main killed it. kill() of a zombie + * succeeds, and the exit code 0 still counts. + */ +static nxt_int_t +nxt_main_store_test_finished(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + int status; + nxt_int_t ret; + nxt_pid_t first, second; + siginfo_t info; + + ret = NXT_ERROR; + + nxt_conf_ver = 0; + nxt_main_store_test_alerts = 0; + + nxt_main_test_store_set_delay(0); + + if (nxt_main_store_test_schedule(task, "{\"c\":3}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the first store"); + } + + first = nxt_main_test_store_pid(); + if (first == 0) { + nxt_main_store_test_fail(thr, "the first store started no child"); + } + + /* Wait for the exit, and leave the child a zombie. */ + + nxt_memzero(&info, sizeof(info)); + + while (waitid(P_PID, first, &info, WEXITED | WNOWAIT) != 0) { + if (nxt_errno != NXT_EINTR) { + nxt_main_store_test_fail(thr, "waitid(%PI) failed %E", first, + nxt_errno); + } + } + + nxt_main_test_store_set_exiting(task, 1); + + if (nxt_main_store_test_schedule(task, "{\"d\":4}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the second store"); + } + + if (nxt_main_store_test_wait(first, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", first, + nxt_errno); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_main_store_test_fail(thr, "the first store failed: status 0x%Xi", + (nxt_uint_t) status); + } + + (void) nxt_main_test_store_exited(task, first, status); + + if (nxt_conf_ver != NXT_VERNUM) { + nxt_main_store_test_fail(thr, "the store of a child that exited " + "with 0 did not count"); + } + + second = nxt_main_test_store_pid(); + if (second == 0) { + nxt_main_store_test_fail(thr, "the pending store did not start"); + } + + if (nxt_main_store_test_wait(second, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", second, + nxt_errno); + } + + (void) nxt_main_test_store_exited(task, second, status); + + if (!nxt_main_store_test_holds(paths->conf, "{\"d\":4}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the last " + "configuration", paths->conf); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + return ret; +} + + +static void +nxt_main_store_test_path(char *buf, const char *dir, const char *name) +{ + (void) nxt_sprintf((u_char *) buf, (u_char *) buf + NXT_MAX_PATH_LEN, + "%s/%s%Z", dir, name); +} + + +nxt_int_t +nxt_main_store_test(nxt_thread_t *thr) +{ + nxt_uint_t conf_ver; + nxt_int_t ret; + nxt_task_t *task; + const char *tmpdir; + nxt_runtime_t rt, *saved_rt; + nxt_main_store_test_paths_t paths; + + nxt_thread_time_update(thr); + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main store test started"); + + task = thr->task; + task->thread = thr; + + tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || tmpdir[0] != '/') { + tmpdir = "/tmp"; + } + + (void) nxt_sprintf((u_char *) paths.dir, + (u_char *) paths.dir + sizeof(paths.dir), + "%s/nxt_main_store_test.XXXXXX%Z", tmpdir); + + if (nxt_slow_path(mkdtemp(paths.dir) == NULL)) { + nxt_log_alert(thr->log, "main store test: mkdtemp(\"%s\") failed %E", + paths.dir, nxt_errno); + return NXT_ERROR; + } + + nxt_main_store_test_path(paths.conf, paths.dir, "conf.json"); + nxt_main_store_test_path(paths.conf_tmp, paths.dir, "conf.json.tmp"); + nxt_main_store_test_path(paths.ver, paths.dir, "version"); + nxt_main_store_test_path(paths.ver_tmp, paths.dir, "version.tmp"); + + nxt_memzero(&rt, sizeof(nxt_runtime_t)); + + rt.state = paths.dir; + rt.conf = paths.conf; + rt.conf_tmp = paths.conf_tmp; + rt.ver = paths.ver; + rt.ver_tmp = paths.ver_tmp; + + saved_rt = thr->runtime; + thr->runtime = &rt; + + conf_ver = nxt_conf_ver; + + nxt_main_store_test_next_handler = thr->log->handler; + thr->log->handler = nxt_main_store_test_log; + + ret = nxt_main_store_test_running(thr, task, &paths); + + if (ret == NXT_OK) { + ret = nxt_main_store_test_cancel(thr, task, &paths); + } + + if (ret == NXT_OK) { + ret = nxt_main_store_test_finished(thr, task, &paths); + } + + thr->log->handler = nxt_main_store_test_next_handler; + thr->runtime = saved_rt; + nxt_conf_ver = conf_ver; + + (void) unlink(paths.conf); + (void) unlink(paths.conf_tmp); + (void) unlink(paths.ver); + (void) unlink(paths.ver_tmp); + (void) rmdir(paths.dir); + + if (ret == NXT_OK) { + nxt_log_error(NXT_LOG_NOTICE, thr->log, "main store test passed"); + } + + return ret; +} diff --git a/src/test/nxt_tests.c b/src/test/nxt_tests.c index 112790639..3602c0156 100644 --- a/src/test/nxt_tests.c +++ b/src/test/nxt_tests.c @@ -365,6 +365,10 @@ main(int argc, char **argv) return 1; } + if (nxt_main_store_test(thr) != NXT_OK) { + return 1; + } + if (nxt_proto_creating_wedge_test(thr) != NXT_OK) { return 1; } diff --git a/src/test/nxt_tests.h b/src/test/nxt_tests.h index 7e3ad9ab8..a5aaaefeb 100644 --- a/src/test/nxt_tests.h +++ b/src/test/nxt_tests.h @@ -100,6 +100,7 @@ nxt_int_t nxt_router_websocket_test(nxt_thread_t *thr); nxt_int_t nxt_router_prepare_msg_test(nxt_thread_t *thr); nxt_int_t nxt_main_start_process_reply_test(nxt_thread_t *thr); nxt_int_t nxt_main_file_store_test(nxt_thread_t *thr); +nxt_int_t nxt_main_store_test(nxt_thread_t *thr); nxt_int_t nxt_proto_creating_wedge_test(nxt_thread_t *thr); nxt_int_t nxt_main_whoami_test(nxt_thread_t *thr); nxt_int_t nxt_main_remove_child_pid_test(nxt_thread_t *thr); diff --git a/test/syscalls/baseline-linux-glibc.txt b/test/syscalls/baseline-linux-glibc.txt index 02bcdbb5f..66e56923f 100644 --- a/test/syscalls/baseline-linux-glibc.txt +++ b/test/syscalls/baseline-linux-glibc.txt @@ -36,6 +36,7 @@ getrandom getsockopt gettid ioctl +kill listen lseek memfd_create diff --git a/test/syscalls/baseline-linux-musl.txt b/test/syscalls/baseline-linux-musl.txt index d005c7901..211f0ba4f 100644 --- a/test/syscalls/baseline-linux-musl.txt +++ b/test/syscalls/baseline-linux-musl.txt @@ -36,6 +36,7 @@ getrandom getsockopt gettid ioctl +kill listen lseek lstat diff --git a/test/syscalls/capture.sh b/test/syscalls/capture.sh index 4d054e298..0b074b9bc 100755 --- a/test/syscalls/capture.sh +++ b/test/syscalls/capture.sh @@ -156,6 +156,17 @@ done grep -q 'syscall drift canary' "$WORK/body.txt" \ || die "served body is not the canary file" +# Main stores the configuration in a child process. Wait for conf.json, so +# that the second PUT below stores over a file that exists. When SIGQUIT +# arrives while a store child runs and a newer store waits, main kills the +# child, and the kill can come before the child created conf.json. +i=0 +while [ ! -s "$WORK/state/conf.json" ]; do + i=$((i + 1)) + [ "$i" -lt 100 ] || die "conf.json was not stored within 10s" + sleep 0.1 +done + # A second PUT, so the capture covers storing over state files that already # exist. The first one writes conf.json and version into an empty statedir, # where nxt_main_file_store_inherit() finds no destination and returns before diff --git a/test/test_state_store.py b/test/test_state_store.py index b7a6c771d..9242b340d 100644 --- a/test/test_state_store.py +++ b/test/test_state_store.py @@ -120,7 +120,14 @@ def test_state_store_full_filesystem(requires_restart, skip_alert): assert 'success' in client.conf(big_conf(400)), 'the large PUT' # The store is attempted asynchronously; wait for it to give up. - Log.wait_for_record(r'failed to store current configuration') + # The store child logs the first alert, and main logs the second + # when it reaps the child. + assert Log.wait_for_record( + r'failed to store current configuration' + ), 'the store child failed' + assert Log.wait_for_record( + r'state store child \d+ failed' + ), 'main logged the failed store' # The store failed; the previously stored configuration is intact, # byte for byte, and still parses. From 8bd4d1dc8c29fe6aa44ec7cba61fc7834866c667 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 30 Sep 2026 03:50:38 +0200 Subject: [PATCH 356/391] main: close inherited descriptors in the state store child The store child is a fork(2) of main, so it holds a copy of every descriptor of main: the listen sockets, the ports and the log files. It needs none of them. A copy of a listen socket keeps the address bound while the child runs, so a listener that the router closes cannot be bound again until the store ends. Now the child closes every descriptor from 3 upwards before it stores. It logs to stderr: nxt_log_time_handler() writes to nxt_stderr, and nxt_runtime_log_files_create() redirects stderr to the log file. It opens the files it writes itself, and the configuration is a mapping that needs no descriptor. The child uses close_range(2) where the libc has it (glibc 2.34, FreeBSD 12.2). On a libc without the wrapper, such as musl, it calls syscall(SYS_close_range). close_range() fails with ENOSYS on Linux before 5.9, and an older seccomp profile can refuse it. Then, on Linux, the child reads /proc/self/fd and closes each descriptor listed there, except the one that reads the directory. It scans again from the start until a scan closes nothing, because a close can make readdir(3) skip an entry. Only when /proc/self/fd cannot be read does a loop close each number up to sysconf(_SC_OPEN_MAX). Where only closefrom(3) exists (FreeBSD before 12.2, NetBSD, OpenBSD, Solaris), the child uses it. auto/files probes for close_range(), the close_range syscall number and closefrom(). The closefrom() probe includes too, because Solaris and illumos declare closefrom() there. glibc 2.34 also has closefrom(). It calls close_range(), then scans /proc/self/fd (__closefrom_fallback() in sysdeps/unix/sysv/linux/closefrom_fallback.c), and it aborts through __fortify_fail() when the scan fails, for example when /proc is not mounted. So the child does not use it where close_range() exists. The cost of the fallback, on pro4s-2 with a limit of 1048576 descriptors and close_range() refused by "strace -f --seccomp-bpf -e inject=close_range:error=EPERM", from the close_range() call to the first unlink(2) of the store, six stores each: - the loop alone: 905.8, 873.1, 876.8, 881.3, 892.6 and 867.9 ms; - with the /proc/self/fd scan: 0.2 ms each. Such a limit is common in containers, and old kernels and old seccomp profiles are where close_range() is missing. The loop is the last resort. Test: src/test/nxt_main_store_test.c holds the read end of a pipe and gives the write end only to the store child. The read end must reach the end of file while the child still sleeps in the test hook, before its store. Before this change, the end of file came only when the child exited. The test passes with close_range(), with syscall(SYS_close_range), with the loop (both forced in a build), and with the /proc/self/fd scan (./build/tests with close_range() refused by strace, six refusals). test/syscalls: both baselines now list close_range. On musl the capture shows the syscall through syscall(SYS_close_range). Refs https://github.com/freeunitorg/freeunit/issues/516. Co-Authored-By: Claude Opus 5.5 --- auto/files | 50 ++++++++++ src/nxt_main_process.c | 128 ++++++++++++++++++++++++ src/test/nxt_main_store_test.c | 129 +++++++++++++++++++++++++ test/syscalls/baseline-linux-glibc.txt | 1 + test/syscalls/baseline-linux-musl.txt | 1 + 5 files changed, 309 insertions(+) diff --git a/auto/files b/auto/files index 4912ed802..6eedead30 100644 --- a/auto/files +++ b/auto/files @@ -69,6 +69,56 @@ nxt_feature_test="#define _GNU_SOURCE . auto/feature +# close_range(), Linux 5.9 with glibc 2.34, FreeBSD 12.2. + +nxt_feature="close_range()" +nxt_feature_name=NXT_HAVE_CLOSE_RANGE +nxt_feature_run= +nxt_feature_incs= +nxt_feature_libs= +nxt_feature_test="#define _GNU_SOURCE + #include + + int main(void) { + return close_range(3, ~0U, 0); + }" +. auto/feature + + +if [ $nxt_found = no ]; then + + # A libc without the wrapper, such as musl, may still have the number. + + nxt_feature="close_range() syscall" + nxt_feature_name=NXT_HAVE_SYS_CLOSE_RANGE + nxt_feature_test="#include + #include + + int main(void) { + return syscall(SYS_close_range, 3, ~0U, 0); + }" + . auto/feature +fi + + +# closefrom(), FreeBSD 8.0, NetBSD 3.0, OpenBSD 3.5, Solaris 9. +# Solaris and illumos declare it in , the others in . + +nxt_feature="closefrom()" +nxt_feature_name=NXT_HAVE_CLOSEFROM +nxt_feature_run= +nxt_feature_incs= +nxt_feature_libs= +nxt_feature_test="#include + #include + + int main(void) { + closefrom(3); + return 0; + }" +. auto/feature + + nxt_feature="openat2()" nxt_feature_name=NXT_HAVE_OPENAT2 nxt_feature_run= diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 6fd89ce03..cfbe7dc74 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -19,6 +19,7 @@ #endif #include +#include typedef struct { @@ -81,6 +82,10 @@ static void nxt_main_port_conf_store_handler(nxt_task_t *task, static void nxt_main_store_schedule(nxt_task_t *task, u_char *p, size_t size); static void nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size); +static void nxt_main_store_close_fds(void); +#if (NXT_LINUX) +static nxt_int_t nxt_main_store_close_proc_fds(void); +#endif static nxt_int_t nxt_main_store_files(nxt_task_t *task, u_char *p, size_t size, nxt_bool_t version); static void nxt_main_start_exit(nxt_task_t *task); @@ -2387,6 +2392,8 @@ nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) nxt_pid = getpid(); task->thread->tid = 0; + nxt_main_store_close_fds(); + #if (NXT_TESTS) if (nxt_main_test_store_delay != 0) { nxt_nanosleep(nxt_main_test_store_delay * 1000000); @@ -2410,6 +2417,127 @@ nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) } +/* + * The store child inherits every descriptor of main: the listen sockets, + * the ports and the log files. A copy of a listen socket keeps the address + * bound while the child runs, so a listener that the router closes cannot + * be bound again until the store ends. The child needs none of them. It + * logs to stderr (nxt_log_time_handler() writes to nxt_stderr, which + * nxt_runtime_log_files_create() redirects to the log file), and it opens + * the files it writes. So it closes every descriptor from 3 upwards. + * + * close_range() fails with ENOSYS on Linux before 5.9, and a seccomp + * profile can refuse it. Then, on Linux, the child closes the descriptors + * that /proc/self/fd lists. Only when that directory cannot be read does + * it close each number up to the limit: with a limit of 1048576 that loop + * took 862 to 892 ms per store. All of this runs in the child, not in + * main. + */ +static void +nxt_main_store_close_fds(void) +{ +#if (NXT_HAVE_CLOSEFROM && !NXT_HAVE_CLOSE_RANGE && !NXT_HAVE_SYS_CLOSE_RANGE) + + closefrom(3); + +#else + + long fd, max; + +#if (NXT_HAVE_CLOSE_RANGE) + + if (close_range(3, ~0U, 0) == 0) { + return; + } + +#elif (NXT_HAVE_SYS_CLOSE_RANGE) + + if (syscall(SYS_close_range, 3, ~0U, 0) == 0) { + return; + } + +#endif + +#if (NXT_LINUX) + + if (nxt_main_store_close_proc_fds() == NXT_OK) { + return; + } + +#endif + + max = sysconf(_SC_OPEN_MAX); + + if (max <= 0) { + max = 1024; + } + + for (fd = 3; fd < max; fd++) { + (void) close((int) fd); + } + +#endif +} + + +#if (NXT_LINUX) + +/* + * Close the descriptors from 3 upwards that /proc/self/fd lists, except + * the one that reads the directory. A close changes the directory while + * it is read, and readdir() can then skip an entry. So the scan runs + * again from the start until a scan closes nothing, as glibc does in + * __closefrom_fallback(). + */ +static nxt_int_t +nxt_main_store_close_proc_fds(void) +{ + int dfd; + DIR *dir; + nxt_int_t fd; + nxt_bool_t closed; + struct dirent *de; + + dir = opendir("/proc/self/fd"); + if (dir == NULL) { + return NXT_ERROR; + } + + dfd = dirfd(dir); + + do { + closed = 0; + + for ( ;; ) { + de = readdir(dir); + if (de == NULL) { + break; + } + + /* "." and ".." do not parse. */ + fd = nxt_int_parse((u_char *) de->d_name, + nxt_strlen(de->d_name)); + + if (fd < 3 || fd == dfd) { + continue; + } + + (void) close((int) fd); + closed = 1; + } + + rewinddir(dir); + + } while (closed); + + (void) closedir(dir); + + return NXT_OK; +} + +#endif + + static nxt_int_t nxt_main_store_files(nxt_task_t *task, u_char *p, size_t size, nxt_bool_t version) diff --git a/src/test/nxt_main_store_test.c b/src/test/nxt_main_store_test.c index b80a209cf..c03332439 100644 --- a/src/test/nxt_main_store_test.c +++ b/src/test/nxt_main_store_test.c @@ -22,6 +22,10 @@ * reaped. * - A child that exited before the signal arrived keeps its exit code: * its store counts, and nxt_conf_ver is set. + * - The store child closes the descriptors it inherits from 3 upwards + * before it stores. The test holds the read end of a pipe and gives + * the write end to the child only. The read end must see the end of + * file while the child still sleeps. */ #include @@ -29,6 +33,7 @@ #include #include "nxt_tests.h" +#include #include #include @@ -521,6 +526,126 @@ nxt_main_store_test_finished(nxt_thread_t *thr, nxt_task_t *task, } +static nxt_msec_t +nxt_main_store_test_now(void) +{ + struct timespec ts; + + (void) clock_gettime(CLOCK_MONOTONIC, &ts); + + return (nxt_msec_t) (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); +} + + +/* + * The child closes the write end of the pipe before its delay. Before the + * change, it held the write end until it exited, after the delay. + */ +static nxt_int_t +nxt_main_store_test_fds(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + int pp[2], n, status; + char c; + ssize_t r; + nxt_int_t ret; + nxt_pid_t pid; + nxt_msec_t start, elapsed; + struct pollfd pfd; + + ret = NXT_ERROR; + + pp[0] = -1; + pp[1] = -1; + + if (pipe(pp) != 0) { + nxt_main_store_test_fail(thr, "pipe() failed %E", nxt_errno); + } + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + start = nxt_main_store_test_now(); + + if (nxt_main_store_test_schedule(task, "{\"e\":5}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the store"); + } + + nxt_main_test_store_set_delay(0); + + pid = nxt_main_test_store_pid(); + if (pid == 0) { + nxt_main_store_test_fail(thr, "the store started no child"); + } + + /* Now only the child can hold the write end. */ + + (void) close(pp[1]); + pp[1] = -1; + + pfd.fd = pp[0]; + pfd.events = POLLIN; + pfd.revents = 0; + + do { + n = poll(&pfd, 1, 4 * NXT_MAIN_STORE_TEST_DELAY); + } while (n == -1 && nxt_errno == NXT_EINTR); + + elapsed = nxt_main_store_test_now() - start; + + if (n != 1) { + nxt_main_store_test_fail(thr, "the pipe did not become readable: " + "poll() returned %d", n); + } + + r = read(pp[0], &c, 1); + + if (r != 0) { + nxt_main_store_test_fail(thr, "read() of the pipe returned %z", r); + } + + if (waitpid(pid, &status, WNOHANG) != 0 + || elapsed >= NXT_MAIN_STORE_TEST_DELAY / 2) + { + nxt_main_store_test_fail(thr, "the store child %PI held an inherited " + "descriptor until it exited (%M ms)", pid, + elapsed); + } + + if (nxt_main_store_test_wait(pid, &status) != 0) { + nxt_main_store_test_fail(thr, "waitpid(%PI) failed %E", pid, + nxt_errno); + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_main_store_test_fail(thr, "the store failed: status 0x%Xi", + (nxt_uint_t) status); + } + + (void) nxt_main_test_store_exited(task, pid, status); + + if (!nxt_main_store_test_holds(paths->conf, "{\"e\":5}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the " + "configuration", paths->conf); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + if (pp[0] != -1) { + (void) close(pp[0]); + } + + if (pp[1] != -1) { + (void) close(pp[1]); + } + + return ret; +} + + static void nxt_main_store_test_path(char *buf, const char *dir, const char *name) { @@ -591,6 +716,10 @@ nxt_main_store_test(nxt_thread_t *thr) ret = nxt_main_store_test_finished(thr, task, &paths); } + if (ret == NXT_OK) { + ret = nxt_main_store_test_fds(thr, task, &paths); + } + thr->log->handler = nxt_main_store_test_next_handler; thr->runtime = saved_rt; nxt_conf_ver = conf_ver; diff --git a/test/syscalls/baseline-linux-glibc.txt b/test/syscalls/baseline-linux-glibc.txt index 66e56923f..6893deb69 100644 --- a/test/syscalls/baseline-linux-glibc.txt +++ b/test/syscalls/baseline-linux-glibc.txt @@ -13,6 +13,7 @@ chmod clone clone3 close +close_range connect dup2 epoll_create diff --git a/test/syscalls/baseline-linux-musl.txt b/test/syscalls/baseline-linux-musl.txt index 211f0ba4f..4defe5215 100644 --- a/test/syscalls/baseline-linux-musl.txt +++ b/test/syscalls/baseline-linux-musl.txt @@ -11,6 +11,7 @@ capset chmod clone close +close_range connect dup2 epoll_create1 From c3d3251fe9051fe0bd783ba24ca21915ad33fb2f Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 30 Sep 2026 04:06:49 +0200 Subject: [PATCH 357/391] main: store certificate bundles in the state store child nxt_cert_store_put_handler() still called nxt_main_file_store() in the event loop of main. Each bundle upload ran two fsync(2) calls there, the same stall that commit b12fb67788838f9ffe829df98adbe4c7a523cf6d removed for conf.json. See https://github.com/freeunitorg/freeunit/issues/516. The store child now runs jobs as well as the conf.json store. A job changes one file in a directory of the state: - NXT_MAIN_STORE_PUT writes "" through ".store.tmp" with nxt_main_file_store(), as before. - NXT_MAIN_STORE_DELETE unlinks "" and flushes the directory. A file that is not there counts as deleted. The interface is in src/nxt_main_process.h. nxt_main_store_job_create() allocates a job and its paths in one block. The caller sets the data of a PUT (nxt_malloc()ed memory that the job owns), and the pid, port id and stream to answer. nxt_main_store_submit() queues the job. Jobs wait in a queue, and each one runs. Only conf.json stores coalesce, as before. One child runs at a time. When a child exits, nxt_main_store_next() starts the next store in the order they came. A newer conf.json store takes the place of the pending store that it replaces. So a certificate DELETE does not run before an older conf.json store that stops naming the bundle. If it ran first, a crash between the two would leave a conf.json that names a deleted bundle. The next start would then log "the previous configuration is invalid: Certificate "a" is not found." and drop the configuration. At exit, nxt_main_store_cancel() kills only a conf.json store child that a pending conf.json store replaces, never a job child. Main answers the sender from the SIGCHLD handler: RPC_READY_LAST when the child exited with 0, RPC_ERROR otherwise. It looks up the port by pid and port id at that time, so a sender that has gone gets no answer. The controller still answers a certificate upload with 200 only when the bundle is on disk. When fork() fails, main runs the job itself and answers at once, as before this change. Main logs a failed job as "failed to store certificate" or "failed to delete certificate", with the name. A certificate store child that something else kills after its rename(2) and before its _exit(0) leaves the new bundle on disk, while the controller answers 500 and restores the old metadata. This case is rare, and it is new with this change: before, main did the store and could not be killed half way without exiting. A controller that exits while a store child runs now starts again only when no store runs or waits. nxt_controller_prefork() reads conf.json and the certificate bundles from disk. Main used to start the new controller at once, so it could read them before the child renamed its file. The new controller then kept the old bundle. Main sent the answer of the job to the controller that had gone, so nothing sent the configuration to the router again. The router got the old bundle with the first configuration of the new controller, and served the old certificate after the store ended. Before this change, main stored a bundle in its event loop, so the store and the restart could not overlap. A conf.json store has the same problem since commit b12fb67788838f9ffe829df98adbe4c7a523cf6d: the new controller sends the old configuration to the router. The wait fixes that too. Only the controller sends stores, so no store is added while it is gone. See https://github.com/freeunitorg/freeunit/pull/528#pullrequestreview-5405819096. The certificate DELETE now is a job too. Main used to unlink the bundle at once. The controller sends no DELETE while an upload waits for its answer, so that order holds today. In the queue it holds in main as well: a delete cannot run before the rename(2) of an earlier store of the same bundle, which would bring the bundle back. Main now also checks the name of a DELETE as it checks the name of an upload: it must end with a NUL inside the message, and it must not be empty, longer than NXT_CERT_NAME_MAX_LENGTH, contain "/" or start with ".". Before, main read it with nxt_strlen() and put it into a path. The size after the name of an upload is read through nxt_span_copy(). The controller: the upload waits for main for as long as the store child runs, not for one handler of main. The controller kept the request at the head of nxt_controller_waiting_requests for that time. When the router restarted then, nxt_controller_conf_init_handler() flushed the queue and ran the upload a second time. It got an early 200, its connection was closed and freed, and when main answered, the reply handler used the freed request, and the controller died. Now the request is not in the queue while main stores it. Instead, nxt_controller_check_postpone_request() makes every other change wait while nxt_controller_store_in_flight() is true. When the store of a bundle that a listener uses ends, the controller sends the configuration to the router again. A router that restarted during the store may still apply its first configuration then (nxt_controller_waiting_init_conf). The router cannot apply two configurations at once: nxt_router_temp_conf() empties the global socket queues for each new configuration, so an apply that waits for an application goes on with the queues of the other one. So the controller now keeps such a store aside, and nxt_controller_conf_init_handler() sends the configuration again when the first one ends. Before main stored bundles in a child, the answer of a store always reached the controller before main handled the READY of a new router, so this could not happen. After this change no fsync(2) runs in the event loop of main, except when fork() fails. Main still runs these without fsync: the unlink(2) of an njs script (nxt_script_store_delete_handler()), the open(2) of an njs script file that the controller then writes (nxt_script_store_put_handler()), and the unlink(2) of a listener's Unix socket. This change does not touch the njs script store. Tests: - src/test/nxt_main_store_test.c, with TLS: an upload through nxt_cert_store_put_handler() leaves nothing on the controller's port while its store child runs. A conf.json store and a second upload arrive meanwhile, and main starts to exit. The certificate child is not killed. Each upload gets RPC_READY_LAST on its own stream, in order, and each bundle is on disk when its answer is there. The conf.json store runs between the two uploads, in the order they came. A DELETE sent right after an upload of the same bundle runs after it, and the bundle is gone at the end. - src/test/nxt_main_store_test.c, the order of stores: a conf.json store A runs. Then come a DELETE of "first", a conf.json store B, a DELETE of "old", and a conf.json store C that replaces B. The stores must run as A, the DELETE of "first", C, the DELETE of "old". With the queued jobs started first, the test fails with '"/old" was deleted before C, which took the place of the older B'. - test/test_tls.py: test_tls_certificate_upload_stored reads each bundle from disk right after its 200, for one upload and for two uploads sent at the same time. - test/test_tls.py: test_tls_certificate_upload_router_restart attaches strace to main and delays each fsync(2) by 2 s. It kills the router while the store child runs, and sends a configuration change after the new router starts. The upload must get its 200 with the bundle on disk, the change must take more than 1 s because it waits for the store, and the controller must survive. With the controller of the earlier version of this change the upload got its answer before the bundle was on disk, and the controller was gone at teardown. The test skips when strace is not installed or cannot attach. - test/test_tls.py: test_tls_certificate_update_router_restart does the same with a bundle that a listener uses, and an application that takes 8 s to start. So the store ends while the new router applies its first configuration. The upload must get "Certificate chain updated." after that configuration, the listener must serve the new certificate, and a later change must succeed. When the controller sent the configuration at once, the upload got "Certificate stored but not applied.". A review of the earlier version also saw main log "bind(...) failed (98: Address already in use)", the listener stop answering, and the next change crash the router with signal 11. - test/test_tls.py: test_tls_certificate_update_controller_restart replaces a bundle that a listener uses, and kills the controller while the store child runs. strace delays each fsync(2) by 2 s. After the store, the new controller must list the new certificate, and the listener must serve it. When main started the controller at once, the test failed with "AssertionError: the controller lists the new certificate" ('old' == 'new'), 3 of 3 runs. Without that assertion it failed with "AssertionError: the router serves the new certificate", because the listener served the certificate with CN "old", 2 of 2 runs. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- CHANGES | 8 + docs/changes.xml | 11 + src/nxt_cert.c | 175 +++++---- src/nxt_controller.c | 118 ++++-- src/nxt_main_process.c | 418 ++++++++++++++++++--- src/nxt_main_process.h | 35 ++ src/test/nxt_main_store_test.c | 666 +++++++++++++++++++++++++++++++++ test/test_tls.py | 398 ++++++++++++++++++++ 8 files changed, 1687 insertions(+), 142 deletions(-) diff --git a/CHANGES b/CHANGES index 0f7baf2b6..a928fd389 100644 --- a/CHANGES +++ b/CHANGES @@ -66,6 +66,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 when a newer configuration waits, it stops the running store first. When the store fails, the main process logs an alert. + *) Change: an uploaded certificate bundle is now also stored by that + child process, so its two fsync(2) calls no longer stop the main + process. The upload is still answered only when the bundle is on + disk. A certificate deletion also runs in the child process, after + every earlier store. A controller process that exits during a store + is started again only when the store ends, so that it reads the + stored files. + *) Bugfix: a regular expression in the configuration could run for up to 10,000,000 match steps on one request. A match now stops after at most 100,000 steps and the request gets a 500 response. The diff --git a/docs/changes.xml b/docs/changes.xml index 2fa669996..592210f1f 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -116,6 +116,17 @@ store first. When the store fails, the main process logs an alert. + + +an uploaded certificate bundle is now also stored by that child process, so +its two fsync(2) calls no longer stop the main process. The upload is still +answered only when the bundle is on disk. A certificate deletion also runs +in the child process, after every earlier store. A controller process that +exits during a store is started again only when the store ends, so that it +reads the stored files. + + + the wasm module did not check the offset that the malloc handler of the guest diff --git a/src/nxt_cert.c b/src/nxt_cert.c index c42477229..accdddcde 100644 --- a/src/nxt_cert.c +++ b/src/nxt_cert.c @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -1664,24 +1665,70 @@ nxt_cert_store_put(nxt_task_t *task, nxt_str_t *name, nxt_buf_mem_t *mbuf, /* - * Main's side of nxt_cert_store_put(). nxt_main_file_store() writes the + * The name of a bundle in a message of the controller: NUL-terminated, not + * empty, at most NXT_CERT_NAME_MAX_LENGTH bytes, without "/", and not + * starting with ".". The controller checks the name already, but it is + * unprivileged, and main builds a path from the name. "span" is left at + * the first byte after the NUL. + */ +static nxt_int_t +nxt_cert_store_name(nxt_buf_t *b, nxt_span_t *span, nxt_str_t *name) +{ + u_char *nul; + const u_char *start; + + if (b == NULL) { + return NXT_ERROR; + } + + nxt_span_init(span, b->mem.pos, b->mem.free); + + nul = memchr(b->mem.pos, '\0', nxt_span_len(span)); + if (nul == NULL) { + return NXT_ERROR; + } + + name->length = nul - b->mem.pos; + + if (nxt_span_take(span, name->length + 1, &start) != 0 + || name->length == 0 + || name->length > NXT_CERT_NAME_MAX_LENGTH + || start[0] == '.' + || memchr(start, '/', name->length) != NULL) + { + return NXT_ERROR; + } + + name->start = b->mem.pos; + + return NXT_OK; +} + + +/* + * Main's side of nxt_cert_store_put(). The store child of main writes the * bundle to "certs/.store.tmp" and renames it over "certs/", so a * reader sees the old bundle or the new one, never a partial file. The * temporary name is fixed, so a name of NAME_MAX bytes still works, and one - * name is enough: the controller sends one bundle at a time, and a bundle - * name cannot start with ".". The controller is unprivileged, so main - * checks the sender and the name again. + * name is enough: one store child runs at a time, and a bundle name cannot + * start with ".". The controller is unprivileged, so main checks the + * sender and the name again. + * + * Main answers when the store child exits (nxt_main_store_submit()), so + * the controller answers 200 only when the bundle is on disk. The two + * fsync(2) calls of the store do not stop main. */ void nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - size_t size, used; - u_char *p, *path, *mem; - nxt_str_t name; - nxt_file_t file; - nxt_port_t *port; - nxt_runtime_t *rt; - nxt_port_msg_type_t type; + size_t size; + u_char *mem; + nxt_str_t name; + nxt_file_t file; + nxt_port_t *port; + nxt_span_t span; + nxt_runtime_t *rt; + nxt_main_store_job_t *job; port = nxt_runtime_port_find(task->thread->runtime, nxt_recv_msg_cmsg_pid(msg), @@ -1696,45 +1743,32 @@ nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; - path = NULL; mem = NULL; - type = NXT_PORT_MSG_RPC_ERROR; if (nxt_slow_path(rt->certs.start == NULL)) { nxt_alert(task, "no certificates storage directory"); - goto done; + goto fail; } if (nxt_slow_path(msg->fd[0] == -1)) { nxt_alert(task, "cert_store_handler: invalid shm fd"); - goto done; + goto fail; } /* The message holds the NUL-terminated name, followed by the size. */ - used = nxt_buf_mem_used_size(&msg->buf->mem); - - name.start = msg->buf->mem.pos; - p = memchr(name.start, '\0', used); - - if (nxt_slow_path(p == NULL - || (size_t) (p - name.start) + 1 + sizeof(size_t) != used - || p == name.start - || (size_t) (p - name.start) > NXT_CERT_NAME_MAX_LENGTH - || name.start[0] == '.' - || memchr(name.start, '/', p - name.start) != NULL)) + if (nxt_slow_path(nxt_cert_store_name(msg->buf, &span, &name) != NXT_OK + || nxt_span_len(&span) != sizeof(size_t))) { nxt_alert(task, "cert_store_handler: invalid certificate name"); - goto done; + goto fail; } - name.length = p - name.start; - - nxt_memcpy(&size, p + 1, sizeof(size_t)); + (void) nxt_span_copy(&span, &size, sizeof(size_t)); if (nxt_slow_path(size == 0 || size > NXT_CERT_STORE_MAX_SIZE)) { nxt_alert(task, "cert_store_handler: invalid bundle size %uz", size); - goto done; + goto fail; } /* @@ -1745,7 +1779,7 @@ nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) mem = nxt_malloc(size); if (nxt_slow_path(mem == NULL)) { - goto done; + goto fail; } nxt_memzero(&file, sizeof(nxt_file_t)); @@ -1756,40 +1790,33 @@ nxt_cert_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) if (nxt_slow_path(nxt_file_read(&file, mem, size, 0) != (ssize_t) size)) { nxt_alert(task, "cert_store_handler: short bundle read"); - goto done; + goto fail; } - /* The buffer holds "/", a NUL, and "/.store.tmp". */ - - path = nxt_malloc(2 * rt->certs.length + name.length + 1 - + nxt_length(".store.tmp") + 1); - if (nxt_slow_path(path == NULL)) { - goto done; + job = nxt_main_store_job_create(NXT_MAIN_STORE_PUT, "certificate", + &rt->certs, &name); + if (nxt_slow_path(job == NULL)) { + goto fail; } - p = nxt_cpymem(path, rt->certs.start, rt->certs.length); - p = nxt_cpymem(p, name.start, name.length + 1); - p = nxt_cpymem(p, rt->certs.start, rt->certs.length); - nxt_memcpy(p, ".store.tmp", nxt_length(".store.tmp") + 1); + job->data = mem; + job->size = size; + job->reply_pid = port->pid; + job->reply_port = port->id; + job->stream = msg->port_msg.stream; - if (nxt_main_file_store(task, (char *) rt->certs.start, - (char *) path + rt->certs.length + name.length + 1, - (char *) path, mem, size) - == NXT_OK) - { - type = NXT_PORT_MSG_RPC_READY_LAST; + nxt_port_recv_msg_close_fds(msg); - } else { - nxt_alert(task, "failed to store certificate \"%V\"", &name); - } + nxt_main_store_submit(task, job); -done: + return; - (void) nxt_port_socket_write(task, port, type, -1, msg->port_msg.stream, - 0, NULL); +fail: + + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, -1, + msg->port_msg.stream, 0, NULL); nxt_free(mem); - nxt_free(path); nxt_port_recv_msg_close_fds(msg); } @@ -1816,14 +1843,20 @@ nxt_cert_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) } +/* + * The store child of main deletes the bundle, after any store that waits + * before it. A delete in main could run before the rename() of an earlier + * store child of the same bundle, and that rename() would bring the bundle + * back. + */ void nxt_cert_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - u_char *p; - nxt_str_t name; - nxt_port_t *ctl_port; - nxt_runtime_t *rt; - nxt_file_name_t *path; + nxt_str_t name; + nxt_port_t *ctl_port; + nxt_span_t span; + nxt_runtime_t *rt; + nxt_main_store_job_t *job; rt = task->thread->runtime; ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; @@ -1846,17 +1879,17 @@ nxt_cert_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - name.start = msg->buf->mem.pos; - name.length = nxt_strlen(name.start); - - path = nxt_malloc(rt->certs.length + name.length + 1); - - if (nxt_fast_path(path != NULL)) { - p = nxt_cpymem(path, rt->certs.start, rt->certs.length); - p = nxt_cpymem(p, name.start, name.length + 1); + if (nxt_slow_path(nxt_cert_store_name(msg->buf, &span, &name) != NXT_OK + || nxt_span_len(&span) != 0)) + { + nxt_alert(task, "cert_delete_handler: invalid certificate name"); + return; + } - (void) nxt_file_delete(path); + job = nxt_main_store_job_create(NXT_MAIN_STORE_DELETE, "certificate", + &rt->certs, &name); - nxt_free(path); + if (nxt_fast_path(job != NULL)) { + nxt_main_store_submit(task, job); } } diff --git a/src/nxt_controller.c b/src/nxt_controller.c index b28cfeb3b..d42cd5901 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -98,6 +98,7 @@ static void nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req); static void nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path); +static nxt_bool_t nxt_controller_store_in_flight(void); static nxt_bool_t nxt_controller_check_postpone_request(nxt_task_t *task); static void nxt_controller_process_status(nxt_task_t *task, nxt_controller_request_t *req); @@ -110,6 +111,8 @@ static void nxt_controller_process_cert(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path); static void nxt_controller_cert_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); +static void nxt_controller_cert_apply(nxt_task_t *task, + nxt_controller_cert_store_t *store); static void nxt_controller_cert_apply_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data); static nxt_bool_t nxt_controller_cert_in_use(nxt_str_t *name); @@ -153,6 +156,8 @@ static nxt_queue_t nxt_controller_waiting_requests; static nxt_bool_t nxt_controller_waiting_init_conf; #if (NXT_TLS) static nxt_bool_t nxt_controller_cert_storing; +/* A stored bundle that waits for the first configuration of a router. */ +static nxt_controller_cert_store_t *nxt_controller_cert_stored; #endif static nxt_conf_value_t *nxt_controller_status; @@ -588,7 +593,10 @@ static void nxt_controller_conf_init_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_runtime_t *rt; + nxt_runtime_t *rt; +#if (NXT_TLS) + nxt_controller_cert_store_t *store; +#endif nxt_controller_waiting_init_conf = 0; @@ -614,6 +622,18 @@ nxt_controller_conf_init_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_controller_listening = 1; } +#if (NXT_TLS) + store = nxt_controller_cert_stored; + + if (store != NULL) { + /* It runs the waiting requests when it ends. */ + nxt_controller_cert_stored = NULL; + + nxt_controller_cert_apply(task, store); + return; + } +#endif + nxt_controller_flush_requests(task); } @@ -1862,6 +1882,25 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, } +/* + * Whether main is storing a file for a request that waits for the answer. + * The store runs in a child of main and can take as long as a few fsync(2) + * calls. The request is not in the waiting queue meanwhile: a router that + * restarts flushes the queue (nxt_controller_conf_init_handler()), and a + * request in it would run a second time, and be freed while main still + * answers it. So the other requests wait while this is true. + */ +static nxt_bool_t +nxt_controller_store_in_flight(void) +{ +#if (NXT_TLS) + return nxt_controller_cert_storing; +#else + return 0; +#endif +} + + static nxt_bool_t nxt_controller_check_postpone_request(nxt_task_t *task) { @@ -1869,6 +1908,7 @@ nxt_controller_check_postpone_request(nxt_task_t *task) nxt_runtime_t *rt; if (!nxt_queue_is_empty(&nxt_controller_waiting_requests) + || nxt_controller_store_in_flight() || nxt_controller_waiting_init_conf || !nxt_controller_router_ready) { @@ -2147,8 +2187,11 @@ nxt_controller_process_cert(nxt_task_t *task, store->name = name; store->req = req; - /* Any other change waits until main answers. */ - nxt_queue_insert_head(&nxt_controller_waiting_requests, &req->link); + /* + * Any other change waits until main answers: see + * nxt_controller_store_in_flight(). The request itself is not in + * the waiting queue, so a flush of the queue cannot run it again. + */ nxt_cert_store_put(task, &name, &c->read->mem, c->mem_pool, nxt_controller_cert_store_handler, store); @@ -2267,42 +2310,73 @@ nxt_controller_process_cert(nxt_task_t *task, /* * Main stored the bundle, or the store failed. On failure, the old metadata - * goes back, so a failed store leaves no metadata without a file. When the - * current configuration names the bundle, the controller sends the - * configuration to the router again. The router reads every bundle during a - * reconfiguration, so new handshakes get the new certificate. Accepted - * connections keep their old TLS context. + * goes back, so a failed store leaves no metadata without a file. */ static void nxt_controller_cert_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_int_t rc; - nxt_runtime_t *rt; - nxt_controller_request_t *req; nxt_controller_response_t resp; nxt_controller_cert_store_t *store; store = data; - req = store->req; - - nxt_queue_remove(&req->link); - - nxt_memzero(&resp, sizeof(nxt_controller_response_t)); nxt_controller_cert_storing = 0; if (msg == NULL || msg->port_msg.type != NXT_PORT_MSG_RPC_READY) { nxt_cert_info_restore(store->info, store->old); + nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + resp.status = 500; resp.title = (u_char *) "Failed to store certificate."; resp.offset = -1; - goto done; + + nxt_controller_response(task, store->req, &resp); + + nxt_controller_flush_requests(task); + return; } nxt_cert_info_release(store->old); + /* + * The router restarted during the store, and the new router applies + * its first configuration now. A second configuration sent before + * that one ends would run at the same time in the router, and the + * router does not support that. nxt_controller_conf_init_handler() + * goes on with this store. The other changes wait meanwhile, because + * nxt_controller_waiting_init_conf is set. + */ + if (nxt_controller_waiting_init_conf) { + nxt_controller_cert_stored = store; + return; + } + + nxt_controller_cert_apply(task, store); +} + + +/* + * When the current configuration names the stored bundle, the controller + * sends the configuration to the router again. The router reads every + * bundle during a reconfiguration, so new handshakes get the new + * certificate. Accepted connections keep their old TLS context. The + * first configuration of a new router may have read the old bundle, so a + * bundle that waited for it is applied as well. + */ +static void +nxt_controller_cert_apply(nxt_task_t *task, nxt_controller_cert_store_t *store) +{ + nxt_int_t rc; + nxt_runtime_t *rt; + nxt_controller_request_t *req; + nxt_controller_response_t resp; + + req = store->req; + + nxt_memzero(&resp, sizeof(nxt_controller_response_t)); + rt = task->thread->runtime; /* When no router is ready, the next router loads the new bundle. */ @@ -2327,13 +2401,11 @@ nxt_controller_cert_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, resp.status = 500; resp.title = (u_char *) "Certificate stored but not applied."; resp.offset = -1; - goto done; - } - - resp.status = 200; - resp.title = (u_char *) "Certificate chain uploaded."; -done: + } else { + resp.status = 200; + resp.title = (u_char *) "Certificate chain uploaded."; + } nxt_controller_response(task, req, &resp); diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index cfbe7dc74..0ed03cb06 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -11,6 +11,7 @@ #include #include #include +#include #if (NXT_TLS) #include #endif @@ -38,11 +39,15 @@ typedef struct { /* The state store runs in a short-lived child of main, one at a time. */ typedef struct { - nxt_pid_t pid; /* The running store child, or 0. */ - nxt_bool_t version; /* It also stores the version file. */ - nxt_bool_t killed; /* Main sent it SIGKILL. */ - u_char *pending; /* The next store's mapping, or NULL. */ - size_t pending_size; + nxt_pid_t pid; /* The running store child, or 0. */ + nxt_bool_t version; /* It also stores the version file. */ + nxt_bool_t killed; /* Main sent it SIGKILL. */ + nxt_main_store_job_t *job; /* Its job, or NULL for conf.json. */ + u_char *pending; /* The next conf.json store, or NULL. */ + size_t pending_size; + nxt_uint_t ahead; /* Queued jobs older than "pending". */ + nxt_main_store_job_t *jobs; /* The jobs that wait, in order. */ + nxt_bool_t controller; /* Start it when the store ends. */ } nxt_main_store_t; @@ -81,7 +86,15 @@ static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); static void nxt_main_store_schedule(nxt_task_t *task, u_char *p, size_t size); +static void nxt_main_store_next(nxt_task_t *task); +static nxt_pid_t nxt_main_store_fork(nxt_task_t *task); static void nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size); +static void nxt_main_store_start_job(nxt_task_t *task, + nxt_main_store_job_t *job); +static nxt_int_t nxt_main_store_job_run(nxt_task_t *task, + nxt_main_store_job_t *job); +static void nxt_main_store_job_done(nxt_task_t *task, + nxt_main_store_job_t *job, nxt_int_t ret); static void nxt_main_store_close_fds(void); #if (NXT_LINUX) static nxt_int_t nxt_main_store_close_proc_fds(void); @@ -1583,6 +1596,18 @@ nxt_main_process_sigchld_handler(nxt_task_t *task, void *obj, void *data) nxt_process_close_ports(task, process); if (init.restart) { + if (init.type == NXT_PROCESS_CONTROLLER + && nxt_main_store.pid != 0) + { + /* See nxt_main_store_exited(). */ + nxt_log(task, NXT_LOG_INFO, "controller starts when " + "state store child %PI ends", + nxt_main_store.pid); + + nxt_main_store.controller = 1; + continue; + } + ret = nxt_process_init_start(task, init); if (nxt_slow_path(ret == NXT_ERROR)) { nxt_alert(task, "failed to restart %s", init.name); @@ -2268,29 +2293,47 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) * handles signals in the meantime (issue #516). * * Only one store child runs at a time: two would race on the same - * temporary name. A store that arrives while a child runs becomes the - * pending store, and a newer store replaces it. nxt_main_store_exited() - * starts the pending store when the child exits. + * temporary name. A conf.json store that arrives while a child runs + * becomes the pending store, and a newer store replaces it. Other stores + * are jobs (nxt_main_store_submit()). They wait in a queue, and each one + * runs. nxt_main_store_next() starts the next store when the child exits. + * + * The stores start in the order they came. A newer conf.json store takes + * the place of the pending store that it replaces. The order matters for + * a certificate DELETE: it must not run before an older conf.json store + * that stops naming the bundle. Otherwise a crash between the two leaves + * a conf.json that names a deleted bundle, and the next start refuses + * that configuration. * * "p" is a mapping of "size" bytes; this function takes ownership of it. */ static void nxt_main_store_schedule(nxt_task_t *task, u_char *p, size_t size) { - if (nxt_main_store.pid == 0) { - nxt_main_store_start(task, p, size); - return; - } + nxt_main_store_job_t *job; if (nxt_main_store.pending != NULL) { nxt_debug(task, "state store: a newer store replaces the pending one"); nxt_mem_munmap(nxt_main_store.pending, nxt_main_store.pending_size); + + } else { + /* The jobs that wait now came first. */ + nxt_main_store.ahead = 0; + + for (job = nxt_main_store.jobs; job != NULL; job = job->next) { + nxt_main_store.ahead++; + } } nxt_main_store.pending = p; nxt_main_store.pending_size = size; + if (nxt_main_store.pid == 0) { + nxt_main_store_next(task); + return; + } + if (nxt_exiting) { nxt_main_store_cancel(task); } @@ -2313,10 +2356,11 @@ nxt_main_start_exit(nxt_task_t *task) /* * At exit, main waits for the store child and then for the pending store. - * When a store is pending, the running child stores a configuration that - * is no longer the last one, so main stops it with SIGKILL and waits for - * one store instead of two. The pending store starts when the child is - * reaped, so two stores still do not overlap. + * When a conf.json store is pending and a conf.json store child runs, that + * child stores a configuration that is no longer the last one, so main + * stops it with SIGKILL and waits for one store instead of two. The next + * store starts when the child is reaped, so two stores still do not + * overlap. Main never kills a job child: each job runs. * * Main does this only at exit. While main runs, the child finishes, and * the pending store waits for it. A kill on each new store would let @@ -2333,6 +2377,7 @@ static void nxt_main_store_cancel(nxt_task_t *task) { if (nxt_main_store.pid == 0 + || nxt_main_store.job != NULL || nxt_main_store.killed || nxt_main_store.pending == NULL) { @@ -2355,6 +2400,193 @@ nxt_main_store_cancel(nxt_task_t *task) } +/* + * Allocate a job that changes "". "dir" ends with "/", its + * "start" is NUL-terminated, and it must live as long as main, as + * rt->certs and rt->scripts do. A PUT job writes through the temporary + * ".store.tmp". One temporary name per directory is enough, because + * only one store runs at a time. So the caller must refuse a "name" that + * starts with ".". "what" names the kind of file in the alerts, for + * example "certificate". + * + * The caller then sets "data" and "size" of a PUT job (nxt_malloc()ed + * memory, which the job owns from here on), and "reply_pid", "reply_port" + * and "stream" when the sender waits for an answer. It gives the job to + * nxt_main_store_submit(), or frees it with nxt_main_store_job_free(). + */ +nxt_main_store_job_t * +nxt_main_store_job_create(nxt_main_store_op_t op, const char *what, + const nxt_str_t *dir, const nxt_str_t *name) +{ + u_char *p; + size_t size, path; + nxt_main_store_job_t *job; + + /* "" and its NUL. */ + if (nxt_size_add(dir->length, name->length, &path) != 0 + || nxt_size_add(path, 1, &path) != 0 + || nxt_size_add(sizeof(nxt_main_store_job_t), path, &size) != 0) + { + return NULL; + } + + /* ".store.tmp" and its NUL. */ + if (op == NXT_MAIN_STORE_PUT + && (nxt_size_add(size, dir->length, &size) != 0 + || nxt_size_add(size, sizeof(NXT_MAIN_STORE_TMP), &size) != 0)) + { + return NULL; + } + + job = nxt_malloc(size); + if (nxt_slow_path(job == NULL)) { + return NULL; + } + + nxt_memzero(job, sizeof(nxt_main_store_job_t)); + + job->op = op; + job->what = what; + job->dir = (const char *) dir->start; + + p = (u_char *) job + sizeof(nxt_main_store_job_t); + + job->name = (char *) p; + job->base = (char *) p + dir->length; + + p = nxt_cpymem(p, dir->start, dir->length); + p = nxt_cpymem(p, name->start, name->length); + *p++ = '\0'; + + if (op == NXT_MAIN_STORE_PUT) { + job->tmp = (char *) p; + + p = nxt_cpymem(p, dir->start, dir->length); + nxt_memcpy(p, NXT_MAIN_STORE_TMP, sizeof(NXT_MAIN_STORE_TMP)); + } + + return job; +} + + +void +nxt_main_store_job_free(nxt_main_store_job_t *job) +{ + nxt_free(job->data); + nxt_free(job); +} + + +/* + * Queue a job after the jobs that wait already. Main answers the sender + * when the job ends: RPC_READY_LAST when the child exited with 0, and + * RPC_ERROR otherwise. When fork() fails, main runs the job itself and + * answers at once. The job is freed after the answer. + */ +void +nxt_main_store_submit(nxt_task_t *task, nxt_main_store_job_t *job) +{ + nxt_main_store_job_t **last; + + job->next = NULL; + + for (last = &nxt_main_store.jobs; *last != NULL; last = &(*last)->next) { + /* void */ + } + + *last = job; + + if (nxt_main_store.pid == 0) { + nxt_main_store_next(task); + } +} + + +/* + * Start stores until a child runs or nothing waits, in the order they came: + * see nxt_main_store_schedule(). A store that cannot fork runs in main, + * and then the next one starts. + */ +static void +nxt_main_store_next(nxt_task_t *task) +{ + u_char *p; + size_t size; + nxt_main_store_job_t *job; + + while (nxt_main_store.pid == 0) { + job = nxt_main_store.jobs; + p = nxt_main_store.pending; + + if (job != NULL && (p == NULL || nxt_main_store.ahead != 0)) { + nxt_main_store.jobs = job->next; + job->next = NULL; + + if (nxt_main_store.ahead != 0) { + nxt_main_store.ahead--; + } + + nxt_main_store_start_job(task, job); + continue; + } + + if (p == NULL) { + return; + } + + size = nxt_main_store.pending_size; + + nxt_main_store.pending = NULL; + nxt_main_store.pending_size = 0; + + nxt_main_store_start(task, p, size); + } +} + + +/* + * Fork a store child. Returns the pid in main, 0 in the child and -1 when + * fork() fails. + * + * The signals of main stay blocked in the child, so SIGTERM or SIGINT sent + * to the process group does not stop the store. The child ends with + * _exit(), which runs no atexit() handler of main. + */ +static nxt_pid_t +nxt_main_store_fork(nxt_task_t *task) +{ + nxt_pid_t pid; + + pid = fork(); + + if (nxt_slow_path(pid < 0)) { + nxt_alert(task, "fork() failed for the state store %E", nxt_errno); + return -1; + } + + if (pid == 0) { + nxt_pid = getpid(); + task->thread->tid = 0; + + nxt_main_store_close_fds(); + +#if (NXT_TESTS) + if (nxt_main_test_store_delay != 0) { + nxt_nanosleep(nxt_main_test_store_delay * 1000000); + } +#endif + + return 0; + } + + nxt_debug(task, "state store child %PI", pid); + + nxt_main_store.pid = pid; + + return pid; +} + + static void nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) { @@ -2367,11 +2599,9 @@ nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) */ version = (nxt_conf_ver != NXT_VERNUM); - pid = fork(); + pid = nxt_main_store_fork(task); if (nxt_slow_path(pid < 0)) { - nxt_alert(task, "fork() failed for the state store %E", nxt_errno); - /* Do not lose the store: do it in main, as before the child. */ if (nxt_main_store_files(task, p, size, version) == NXT_OK && version) @@ -2384,22 +2614,6 @@ nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) } if (pid == 0) { - /* - * The signals of main stay blocked, so SIGTERM or SIGINT sent to - * the process group does not stop the store. _exit() runs no - * atexit() handler of main. - */ - nxt_pid = getpid(); - task->thread->tid = 0; - - nxt_main_store_close_fds(); - -#if (NXT_TESTS) - if (nxt_main_test_store_delay != 0) { - nxt_nanosleep(nxt_main_test_store_delay * 1000000); - } -#endif - if (nxt_main_store_files(task, p, size, version) != NXT_OK) { _exit(1); } @@ -2407,16 +2621,105 @@ nxt_main_store_start(nxt_task_t *task, u_char *p, size_t size) _exit(0); } - nxt_debug(task, "state store child %PI", pid); - /* The child has its own copy of the mapping. */ nxt_mem_munmap(p, size); - nxt_main_store.pid = pid; nxt_main_store.version = version; } +static void +nxt_main_store_start_job(nxt_task_t *task, nxt_main_store_job_t *job) +{ + nxt_pid_t pid; + + pid = nxt_main_store_fork(task); + + if (nxt_slow_path(pid < 0)) { + nxt_main_store_job_done(task, job, nxt_main_store_job_run(task, job)); + return; + } + + if (pid == 0) { + if (nxt_main_store_job_run(task, job) != NXT_OK) { + _exit(1); + } + + _exit(0); + } + + /* The child has its own copy of the data. */ + nxt_free(job->data); + job->data = NULL; + + nxt_main_store.job = job; +} + + +static nxt_int_t +nxt_main_store_job_run(nxt_task_t *task, nxt_main_store_job_t *job) +{ + if (job->op == NXT_MAIN_STORE_PUT) { + return nxt_main_file_store(task, job->dir, job->tmp, job->name, + job->data, job->size); + } + + /* A file that is not there is deleted already. */ + if (unlink(job->name) != 0) { + if (nxt_errno == NXT_ENOENT) { + return NXT_OK; + } + + nxt_alert(task, "unlink(\"%s\") failed %E", job->name, nxt_errno); + + return NXT_ERROR; + } + + /* The file is gone; a failed flush only costs durability. */ + (void) nxt_file_dir_sync(task, (nxt_file_name_t *) job->dir); + + return NXT_OK; +} + + +/* + * Log a failed job, answer the sender and free the job. Main logs the + * failure: the alerts of the child can go to a rotated log, because + * SIGUSR1 reopens the log in main only. A sender that has gone has no + * port in the runtime any more, and gets no answer. + */ +static void +nxt_main_store_job_done(nxt_task_t *task, nxt_main_store_job_t *job, + nxt_int_t ret) +{ + nxt_port_t *port; + nxt_port_msg_type_t type; + + if (ret == NXT_OK) { + type = NXT_PORT_MSG_RPC_READY_LAST; + + } else { + type = NXT_PORT_MSG_RPC_ERROR; + + nxt_alert(task, "failed to %s %s \"%s\"", + (job->op == NXT_MAIN_STORE_PUT) ? "store" : "delete", + job->what, job->base); + } + + if (job->reply_pid != 0) { + port = nxt_runtime_port_find(task->thread->runtime, job->reply_pid, + job->reply_port); + + if (port != NULL) { + (void) nxt_port_socket_write(task, port, type, -1, job->stream, + 0, NULL); + } + } + + nxt_main_store_job_free(job); +} + + /* * The store child inherits every descriptor of main: the listen sockets, * the ports and the log files. A copy of a listen socket keeps the address @@ -2592,18 +2895,34 @@ nxt_main_store_cancelled(nxt_pid_t pid, int status) * rotated log, because SIGUSR1 reopens the log in main only. A child that * main killed is not a failure: a newer store is pending. A child that * exited before the signal arrived keeps its exit code. + * + * A controller that exits while a store child runs starts again here, when + * no store runs or waits. nxt_controller_prefork() reads conf.json and the + * certificate bundles from disk. Started at once, the new controller could + * read them before the child renames its file. It would then keep the old + * state, and main would send the answer of the store to the controller that + * has gone. Only the controller sends stores, so no store is added after + * main has reaped it. */ static nxt_bool_t nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) { - u_char *p; - size_t size; + nxt_int_t ret; + nxt_bool_t ok; + nxt_main_store_job_t *job; if (pid != nxt_main_store.pid) { return 0; } - if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { + ok = (WIFEXITED(status) && WEXITSTATUS(status) == 0); + + job = nxt_main_store.job; + + if (job != NULL) { + nxt_main_store_job_done(task, job, ok ? NXT_OK : NXT_ERROR); + + } else if (ok) { if (nxt_main_store.version) { nxt_conf_ver = NXT_VERNUM; } @@ -2619,16 +2938,19 @@ nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) nxt_main_store.pid = 0; nxt_main_store.killed = 0; + nxt_main_store.job = NULL; - p = nxt_main_store.pending; - - if (p != NULL) { - size = nxt_main_store.pending_size; + nxt_main_store_next(task); - nxt_main_store.pending = NULL; - nxt_main_store.pending_size = 0; + if (nxt_main_store.pid == 0 && nxt_main_store.controller) { + nxt_main_store.controller = 0; - nxt_main_store_start(task, p, size); + if (!nxt_exiting) { + ret = nxt_process_init_start(task, nxt_controller_process); + if (nxt_slow_path(ret == NXT_ERROR)) { + nxt_alert(task, "failed to restart controller"); + } + } } return 1; diff --git a/src/nxt_main_process.h b/src/nxt_main_process.h index 9c1af277a..682aba9d9 100644 --- a/src/nxt_main_process.h +++ b/src/nxt_main_process.h @@ -25,6 +25,41 @@ nxt_int_t nxt_main_file_store(nxt_task_t *task, const char *dir, const char *tmp_name, const char *name, u_char *buf, size_t size); +/* + * A change of a file in a directory of the state, run by the store child + * of main. The conf.json store is not a job. See nxt_main_store_submit(). + */ + +#define NXT_MAIN_STORE_TMP ".store.tmp" + +typedef enum { + NXT_MAIN_STORE_PUT = 0, + NXT_MAIN_STORE_DELETE, +} nxt_main_store_op_t; + +typedef struct nxt_main_store_job_s nxt_main_store_job_t; + +struct nxt_main_store_job_s { + nxt_main_store_job_t *next; + nxt_main_store_op_t op; + const char *what; /* For the alerts: "certificate". */ + const char *dir; /* Flushed after the change. */ + char *name; /* "". */ + const char *base; + char *tmp; /* PUT: ".store.tmp". */ + u_char *data; /* PUT: nxt_malloc()ed, owned. */ + size_t size; + nxt_pid_t reply_pid; /* 0: nobody waits for an answer. */ + nxt_port_id_t reply_port; + uint32_t stream; +}; + +nxt_main_store_job_t *nxt_main_store_job_create(nxt_main_store_op_t op, + const char *what, const nxt_str_t *dir, const nxt_str_t *name); +void nxt_main_store_job_free(nxt_main_store_job_t *job); +void nxt_main_store_submit(nxt_task_t *task, nxt_main_store_job_t *job); + + NXT_EXPORT extern nxt_uint_t nxt_conf_ver; NXT_EXPORT extern const nxt_process_init_t nxt_discovery_process; NXT_EXPORT extern const nxt_process_init_t nxt_controller_process; diff --git a/src/test/nxt_main_store_test.c b/src/test/nxt_main_store_test.c index c03332439..f8e1da78b 100644 --- a/src/test/nxt_main_store_test.c +++ b/src/test/nxt_main_store_test.c @@ -26,11 +26,28 @@ * before it stores. The test holds the read end of a pipe and gives * the write end to the child only. The read end must see the end of * file while the child still sleeps. + * - Stores start in the order they came. A conf.json store that + * replaces a pending one takes its place. So a DELETE never runs + * before an older conf.json store. + * - Certificate bundles (with TLS). nxt_cert_store_put_handler() gives + * the bundle to a store child and answers the controller only when the + * child exits: nothing is on the reply port while the child runs. Two + * uploads in quick succession both land, each gets its own answer, in + * order. A conf.json store that arrives between them runs between + * them, and it does not kill the certificate child, even when main + * exits. A DELETE that follows a PUT of the same bundle runs after + * it, so the bundle is gone at the end. */ #include #include +#include +#include #include +#include +#if (NXT_TLS) +#include +#endif #include "nxt_tests.h" #include @@ -54,6 +71,8 @@ typedef struct { char conf_tmp[NXT_MAX_PATH_LEN]; char ver[NXT_MAX_PATH_LEN]; char ver_tmp[NXT_MAX_PATH_LEN]; + char certs[NXT_MAX_PATH_LEN]; + char bundle[NXT_MAX_PATH_LEN]; } nxt_main_store_test_paths_t; @@ -646,6 +665,641 @@ nxt_main_store_test_fds(nxt_thread_t *thr, nxt_task_t *task, } +static nxt_int_t +nxt_main_store_test_write(const char *name, const char *content) +{ + int fd; + size_t len; + ssize_t n; + + fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd == -1) { + return NXT_ERROR; + } + + len = nxt_strlen(content); + + n = write(fd, content, len); + + close(fd); + + return (n == (ssize_t) len) ? NXT_OK : NXT_ERROR; +} + + +/* Reap the store child that runs now, and require that it exited with 0. */ +static nxt_int_t +nxt_main_store_test_reap(nxt_thread_t *thr, nxt_task_t *task, + const char *what) +{ + int status; + nxt_pid_t pid; + + pid = nxt_main_test_store_pid(); + if (pid == 0) { + nxt_log_alert(thr->log, "main store test: no store child for %s", + what); + return NXT_ERROR; + } + + if (nxt_main_store_test_wait(pid, &status) != 0) { + nxt_log_alert(thr->log, "main store test: waitpid(%PI) failed %E", + pid, nxt_errno); + return NXT_ERROR; + } + + (void) nxt_main_test_store_exited(task, pid, status); + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + nxt_log_alert(thr->log, "main store test: the store child of %s " + "ended with status 0x%Xi", what, (nxt_uint_t) status); + return NXT_ERROR; + } + + return NXT_OK; +} + + +/* "" exists. */ +static nxt_bool_t +nxt_main_store_test_exists(const char *dir, const char *name) +{ + char path[NXT_MAX_PATH_LEN]; + struct stat st; + + (void) nxt_sprintf((u_char *) path, (u_char *) path + sizeof(path), + "%s%s%Z", dir, name); + + return (stat(path, &st) == 0); +} + + +/* Queue a DELETE of "" that nobody waits for. */ +static nxt_int_t +nxt_main_store_test_submit_delete(nxt_task_t *task, const char *dir, + const char *name) +{ + nxt_str_t d, n; + nxt_main_store_job_t *job; + + d.start = (u_char *) dir; + d.length = nxt_strlen(dir); + n.start = (u_char *) name; + n.length = nxt_strlen(name); + + job = nxt_main_store_job_create(NXT_MAIN_STORE_DELETE, "file", &d, &n); + if (nxt_slow_path(job == NULL)) { + return NXT_ERROR; + } + + nxt_main_store_submit(task, job); + + return NXT_OK; +} + + +/* + * Stores start in the order they came. A conf.json store "A" runs. Then + * come a DELETE of "first", a conf.json store "B", a DELETE of "old", and a + * conf.json store "C" that replaces "B". The order must be A, the DELETE + * of "first", C, the DELETE of "old". Before the change, both DELETEs ran + * before C, so "old" was gone while conf.json still held A. + */ +static nxt_int_t +nxt_main_store_test_order(nxt_thread_t *thr, nxt_task_t *task, + nxt_main_store_test_paths_t *paths) +{ + nxt_int_t ret; + char dir[NXT_MAX_PATH_LEN], name[NXT_MAX_PATH_LEN]; + + ret = NXT_ERROR; + + nxt_main_store_test_alerts = 0; + + nxt_main_test_store_set_delay(0); + + (void) nxt_sprintf((u_char *) dir, (u_char *) dir + sizeof(dir), "%s/%Z", + paths->dir); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sfirst%Z", dir); + + if (nxt_main_store_test_write(name, "first") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot write \"%s\"", name); + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sold%Z", dir); + + if (nxt_main_store_test_write(name, "old") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot write \"%s\"", name); + } + + /* The child of A runs; the rest waits for it. */ + + if (nxt_main_store_test_schedule(task, "{\"A\":1}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map A"); + } + + if (nxt_main_test_store_pid() == 0) { + nxt_main_store_test_fail(thr, "A started no child"); + } + + if (nxt_main_store_test_submit_delete(task, dir, "first") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot queue the DELETE of first"); + } + + if (nxt_main_store_test_schedule(task, "{\"B\":2}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map B"); + } + + if (nxt_main_store_test_submit_delete(task, dir, "old") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot queue the DELETE of old"); + } + + if (nxt_main_store_test_schedule(task, "{\"C\":3}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map C"); + } + + if (nxt_main_store_test_reap(thr, task, "A") != NXT_OK) { + goto done; + } + + if (!nxt_main_store_test_holds(paths->conf, "{\"A\":1}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold A", paths->conf); + } + + /* The DELETE of "first" came before B, so it runs before C. */ + + if (nxt_main_store_test_reap(thr, task, "the second store") != NXT_OK) { + goto done; + } + + if (nxt_main_store_test_exists(dir, "first") + || !nxt_main_store_test_holds(paths->conf, "{\"A\":1}")) + { + nxt_main_store_test_fail(thr, "the second store was not the DELETE " + "of \"first\", which came before B"); + } + + /* C took the place of B, so it runs before the DELETE of "old". */ + + if (nxt_main_store_test_reap(thr, task, "the third store") != NXT_OK) { + goto done; + } + + if (!nxt_main_store_test_exists(dir, "old")) { + nxt_main_store_test_fail(thr, "\"%sold\" was deleted before C, which " + "took the place of the older B", dir); + } + + if (!nxt_main_store_test_holds(paths->conf, "{\"C\":3}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold C after the " + "third store", paths->conf); + } + + if (nxt_main_store_test_reap(thr, task, "the DELETE of old") != NXT_OK) { + goto done; + } + + if (nxt_main_store_test_exists(dir, "old")) { + nxt_main_store_test_fail(thr, "\"%sold\" was not deleted", dir); + } + + if (nxt_main_test_store_pid() != 0) { + nxt_main_store_test_fail(thr, "a store child runs with nothing " + "pending"); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sfirst%Z", dir); + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sold%Z", dir); + (void) unlink(name); + + return ret; +} + + +#if (NXT_TLS) + +/* A message of the controller, as the port layer gives it to main. */ +static void +nxt_main_store_test_msg(nxt_port_recv_msg_t *msg, nxt_buf_t *b, u_char *buf, + size_t len, nxt_port_t *port, uint32_t stream, nxt_fd_t fd) +{ + nxt_memzero(msg, sizeof(nxt_port_recv_msg_t)); + nxt_memzero(b, sizeof(nxt_buf_t)); + + b->mem.start = buf; + b->mem.pos = buf; + b->mem.free = buf + len; + b->mem.end = buf + len; + + msg->buf = b; + msg->fd[0] = fd; + msg->fd[1] = -1; + msg->port_msg.pid = nxt_pid; + msg->port_msg.reply_port = port->id; + msg->port_msg.stream = stream; +#if (NXT_USE_CMSG_PID) + msg->cmsg_pid = nxt_pid; +#endif +} + + +/* PUT /certificates/, as nxt_cert_store_put() sends it. */ +static nxt_int_t +nxt_main_store_test_put(nxt_task_t *task, nxt_main_store_test_paths_t *paths, + nxt_port_t *port, const char *name, const char *content, uint32_t stream) +{ + u_char *p; + size_t size; + nxt_fd_t fd; + nxt_buf_t b; + nxt_port_recv_msg_t msg; + u_char buf[64]; + + if (nxt_main_store_test_write(paths->bundle, content) != NXT_OK) { + return NXT_ERROR; + } + + fd = open(paths->bundle, O_RDONLY); + if (fd == -1) { + return NXT_ERROR; + } + + size = nxt_strlen(content); + + p = nxt_cpymem(buf, name, nxt_strlen(name) + 1); + p = nxt_cpymem(p, &size, sizeof(size_t)); + + nxt_main_store_test_msg(&msg, &b, buf, p - buf, port, stream, fd); + + nxt_cert_store_put_handler(task, &msg); + + if (msg.fd[0] != -1 && nxt_test_fd_is_open(fd)) { + (void) close(fd); + } + + return NXT_OK; +} + + +/* DELETE /certificates/, as nxt_cert_store_delete() sends it. */ +static void +nxt_main_store_test_delete(nxt_task_t *task, nxt_port_t *port, + const char *name) +{ + size_t len; + nxt_buf_t b; + nxt_port_recv_msg_t msg; + u_char buf[64]; + + len = nxt_strlen(name) + 1; + + nxt_memcpy(buf, name, len); + + nxt_main_store_test_msg(&msg, &b, buf, len, port, 0, -1); + + nxt_cert_store_delete_handler(task, &msg); +} + + +/* The number of answers on the port, and the last one. */ +static nxt_uint_t +nxt_main_store_test_replies(nxt_port_t *port, nxt_port_send_msg_t **last) +{ + nxt_uint_t n; + nxt_queue_link_t *link; + + n = 0; + *last = NULL; + + for (link = nxt_queue_first(&port->messages); + link != nxt_queue_tail(&port->messages); + link = nxt_queue_next(link)) + { + n++; + *last = nxt_queue_link_data(link, nxt_port_send_msg_t, link); + } + + return n; +} + + +static nxt_int_t +nxt_main_store_test_reply(nxt_thread_t *thr, nxt_port_t *port, nxt_uint_t n, + uint32_t stream, const char *what) +{ + nxt_uint_t count; + nxt_port_send_msg_t *sent; + + count = nxt_main_store_test_replies(port, &sent); + + if (count != n) { + nxt_log_alert(thr->log, "main store test: %ui answers after %s " + "(expected %ui)", count, what, n); + return NXT_ERROR; + } + + if (sent->port_msg.type != _NXT_PORT_MSG_RPC_READY + || sent->port_msg.last != 1 + || sent->port_msg.stream != stream) + { + nxt_log_alert(thr->log, "main store test: %s answered type %d, " + "last %d, stream %uD (expected %d, 1, %uD)", what, + (int) sent->port_msg.type, (int) sent->port_msg.last, + sent->port_msg.stream, (int) _NXT_PORT_MSG_RPC_READY, + stream); + return NXT_ERROR; + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_main_store_test_certs(nxt_thread_t *thr, nxt_task_t *task, + nxt_runtime_t *rt, nxt_main_store_test_paths_t *paths) +{ + char name[NXT_MAX_PATH_LEN]; + nxt_mp_t *mp; + nxt_int_t ret; + nxt_port_t *port; + struct stat st; + nxt_event_engine_t engine, *saved_engine; + nxt_port_send_msg_t *sent; + + ret = NXT_ERROR; + port = NULL; + + if (mkdir(paths->certs, 0700) != 0) { + nxt_log_alert(thr->log, "main store test: mkdir(\"%s\") failed %E", + paths->certs, nxt_errno); + return NXT_ERROR; + } + + rt->certs.start = (u_char *) paths->certs; + rt->certs.length = nxt_strlen(paths->certs); + + mp = nxt_mp_create(1024, 128, 256, 32); + if (nxt_slow_path(mp == NULL)) { + return NXT_ERROR; + } + + /* The engine is what the teardown of queued messages needs. */ + nxt_memzero(&engine, sizeof(engine)); + nxt_work_queue_cache_create(&engine.work_queue_cache, 1024); + engine.fast_work_queue.cache = &engine.work_queue_cache; + nxt_work_queue_name(&engine.fast_work_queue, "fast"); + engine.mem_pool = mp; + + saved_engine = thr->engine; + thr->engine = &engine; + rt->main_engine = &engine; + + /* + * The port of the controller. No queue and write_ready unset, so an + * answer is queued on port->messages rather than written. + */ + port = nxt_port_new(task, 0, nxt_pid, NXT_PROCESS_CONTROLLER); + if (nxt_slow_path(port == NULL)) { + goto done; + } + + port->pair[0] = -1; + port->pair[1] = -1; + port->socket.fd = -1; + + rt->port_by_type[NXT_PROCESS_CONTROLLER] = port; + + if (nxt_slow_path(nxt_port_hash_add(&rt->ports, port) != NXT_OK)) { + nxt_port_use(task, port, -1); + port = NULL; + goto done; + } + + nxt_main_store_test_alerts = 0; + + /* The first upload. Its child sleeps before it stores. */ + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + if (nxt_main_store_test_put(task, paths, port, "one", "bundle one", + 0x11111111) + != NXT_OK) + { + nxt_main_store_test_fail(thr, "cannot send the first bundle"); + } + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_main_store_test_fail(thr, "main answered the upload before the " + "bundle was stored"); + } + + if (nxt_main_test_store_pid() == 0) { + nxt_main_store_test_fail(thr, "the upload started no store child"); + } + + /* + * A conf.json store and a second upload arrive while it runs. Main + * starts to exit then, so a conf.json store child would be killed. + */ + + if (nxt_main_store_test_schedule(task, "{\"f\":6}") != NXT_OK) { + nxt_main_store_test_fail(thr, "cannot map the conf.json store"); + } + + nxt_main_test_store_set_exiting(task, 1); + + if (nxt_main_store_test_put(task, paths, port, "two", "bundle two", + 0x22222222) + != NXT_OK) + { + nxt_main_store_test_fail(thr, "cannot send the second bundle"); + } + + nxt_main_test_store_set_delay(0); + + if (!nxt_queue_is_empty(&port->messages)) { + nxt_main_store_test_fail(thr, "main answered an upload before its " + "store child exited"); + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sone%Z", paths->certs); + + if (nxt_main_store_test_reap(thr, task, "the first bundle") != NXT_OK) { + goto done; + } + + if (nxt_main_store_test_reply(thr, port, 1, 0x11111111, + "the first bundle") + != NXT_OK) + { + goto done; + } + + if (!nxt_main_store_test_holds(name, "bundle one")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the first bundle " + "after the answer", name); + } + + /* The conf.json store came before the second upload. */ + + if (nxt_main_store_test_reap(thr, task, "conf.json") != NXT_OK) { + goto done; + } + + if (!nxt_main_store_test_holds(paths->conf, "{\"f\":6}")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the " + "configuration", paths->conf); + } + + if (nxt_main_store_test_replies(port, &sent) != 1) { + nxt_main_store_test_fail(thr, "the second bundle was stored before " + "the older conf.json store"); + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%stwo%Z", paths->certs); + + if (nxt_main_store_test_reap(thr, task, "the second bundle") != NXT_OK) { + goto done; + } + + if (nxt_main_store_test_reply(thr, port, 2, 0x22222222, + "the second bundle") + != NXT_OK) + { + goto done; + } + + if (!nxt_main_store_test_holds(name, "bundle two")) { + nxt_main_store_test_fail(thr, "\"%s\" does not hold the second " + "bundle after the answer", name); + } + + if (nxt_main_test_store_pid() != 0) { + nxt_main_store_test_fail(thr, "a store child runs with nothing " + "pending"); + } + + nxt_main_test_store_set_exiting(task, 0); + + /* A DELETE right after a PUT of the same bundle runs after it. */ + + nxt_main_test_store_set_delay(NXT_MAIN_STORE_TEST_DELAY); + + if (nxt_main_store_test_put(task, paths, port, "three", "bundle three", + 0x33333333) + != NXT_OK) + { + nxt_main_store_test_fail(thr, "cannot send the third bundle"); + } + + nxt_main_test_store_set_delay(0); + + nxt_main_store_test_delete(task, port, "three"); + + if (nxt_main_store_test_reap(thr, task, "the third bundle") != NXT_OK) { + goto done; + } + + /* A delete that main ran at once started no child. */ + if (nxt_main_test_store_pid() != 0 + && nxt_main_store_test_reap(thr, task, "the delete") != NXT_OK) + { + goto done; + } + + if (nxt_main_store_test_reply(thr, port, 3, 0x33333333, + "the third bundle") + != NXT_OK) + { + goto done; + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sthree%Z", paths->certs); + + if (stat(name, &st) == 0) { + nxt_main_store_test_fail(thr, "\"%s\" is back: the delete ran before " + "the store", name); + } + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%s" NXT_MAIN_STORE_TMP "%Z", paths->certs); + + if (stat(name, &st) == 0) { + nxt_main_store_test_fail(thr, "\"%s\" was left behind", name); + } + + if (nxt_main_store_test_alerts != 0) { + nxt_main_store_test_fail(thr, "%ui alerts", nxt_main_store_test_alerts); + } + + ret = NXT_OK; + +done: + + nxt_main_store_test_reap_all(task); + + if (port != NULL) { + /* Frees the queued answers and their port references. */ + nxt_port_test_run_error_handler(task, port); + + (void) nxt_port_hash_remove(&rt->ports, port); + rt->port_by_type[NXT_PROCESS_CONTROLLER] = NULL; + + nxt_port_use(task, port, -1); + } + + thr->engine = saved_engine; + rt->main_engine = NULL; + + nxt_work_queue_cache_destroy(&engine.work_queue_cache); + nxt_mp_destroy(mp); + + (void) unlink(paths->bundle); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sone%Z", paths->certs); + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%stwo%Z", paths->certs); + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%sthree%Z", paths->certs); + (void) unlink(name); + + (void) nxt_sprintf((u_char *) name, (u_char *) name + sizeof(name), + "%s" NXT_MAIN_STORE_TMP "%Z", paths->certs); + (void) unlink(name); + + (void) rmdir(paths->certs); + + return ret; +} + +#endif /* NXT_TLS */ + + static void nxt_main_store_test_path(char *buf, const char *dir, const char *name) { @@ -689,6 +1343,8 @@ nxt_main_store_test(nxt_thread_t *thr) nxt_main_store_test_path(paths.conf_tmp, paths.dir, "conf.json.tmp"); nxt_main_store_test_path(paths.ver, paths.dir, "version"); nxt_main_store_test_path(paths.ver_tmp, paths.dir, "version.tmp"); + nxt_main_store_test_path(paths.certs, paths.dir, "certs/"); + nxt_main_store_test_path(paths.bundle, paths.dir, "bundle"); nxt_memzero(&rt, sizeof(nxt_runtime_t)); @@ -720,6 +1376,16 @@ nxt_main_store_test(nxt_thread_t *thr) ret = nxt_main_store_test_fds(thr, task, &paths); } + if (ret == NXT_OK) { + ret = nxt_main_store_test_order(thr, task, &paths); + } + +#if (NXT_TLS) + if (ret == NXT_OK) { + ret = nxt_main_store_test_certs(thr, task, &rt, &paths); + } +#endif + thr->log->handler = nxt_main_store_test_next_handler; thr->runtime = saved_rt; nxt_conf_ver = conf_ver; diff --git a/test/test_tls.py b/test/test_tls.py index 0fd49a2ef..c61b6fe6b 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -5,6 +5,7 @@ import ssl import subprocess import time +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest @@ -428,6 +429,403 @@ def test_tls_certificate_update_store_fail(skip_alert): shutil.rmtree(tmp) +def test_tls_certificate_upload_stored(): + """The 200 of an upload comes only when main has stored the bundle. + Main stores it in a child process + (https://github.com/freeunitorg/freeunit/issues/516) and answers when + the child exits. Two uploads sent at the same time both land.""" + + names = ('first', 'second', 'third') + + bundles = {} + + for name in names: + client.certificate(name, False) + + with open(f'{option.temp_dir}/{name}.key', 'rb') as k, open( + f'{option.temp_dir}/{name}.crt', 'rb' + ) as c: + bundles[name] = k.read() + c.read() + + certs = Path(f'{option.temp_dir}/state/certs') + + # One upload. The file is complete when the answer arrives. + assert 'success' in client.conf( + bundles['first'], '/certificates/first' + ), 'first uploaded' + assert (certs / 'first').read_bytes() == bundles['first'], 'first stored' + + # Two uploads at the same time. The controller runs one after the + # other, and main stores each one. + def upload(name): + return ApplicationTLS().conf(bundles[name], f'/certificates/{name}') + + with ThreadPoolExecutor(max_workers=2) as pool: + answers = list(pool.map(upload, ('second', 'third'))) + + for name, answer in zip(('second', 'third'), answers): + assert 'success' in answer, f'{name} uploaded' + assert (certs / name).read_bytes() == bundles[name], f'{name} stored' + + assert set(names) <= set(client.conf_get('/certificates')), 'listed' + assert not (certs / '.store.tmp').exists(), 'no temporary file' + + +def unit_children(unit_pid, name): + out = subprocess.check_output( + ['ps', 'ax', '-o', 'pid=,ppid=,args='] + ).decode() + return [ + f[0] + for f in (line.split(None, 2) for line in out.splitlines()) + if len(f) == 3 and f[1] == str(unit_pid) and name in f[2] + ] + + +def trace_fsync(unit_pid): + """Delay each fsync(2) of main and its children by 2 s with strace. + Skip the test when strace is not installed or cannot attach.""" + + strace = shutil.which('strace') + if strace is None: + pytest.skip('strace is not installed') + + tracer = subprocess.Popen( + [ + strace, + '-f', + '-p', + str(unit_pid), + '-o', + '/dev/null', + '-e', + 'trace=fsync', + '-e', + 'inject=fsync:delay_exit=2000000', + ], + stderr=subprocess.PIPE, + ) + + # strace says "Process N attached" on stderr once it traces main. + line = tracer.stderr.readline().decode() + if 'attached' not in line: + tracer.kill() + tracer.wait() + pytest.skip(f'strace cannot attach: {line.strip()}') + + return tracer + + +def test_tls_certificate_upload_router_restart( + skip_alert, skip_fds_check, unit_pid +): + """A router that restarts while main stores a bundle does not make the + controller run the upload again. The upload gets one 200, a change + sent during the store runs after it, and the controller survives. + + strace delays each fsync(2) of main and its children by 2 s, so the + store child of the upload runs for about 4 s.""" + + def children(name): + return unit_children(unit_pid, name) + + client.certificate('slow', False) + + with open(f'{option.temp_dir}/slow.key', 'rb') as k, open( + f'{option.temp_dir}/slow.crt', 'rb' + ) as c: + bundle = k.read() + c.read() + + controller = children('unit: controller') + router = children('unit: router') + assert len(controller) == 1 and len(router) == 1, 'processes' + + tracer = trace_fsync(unit_pid) + + done = {} + stored = Path(f'{option.temp_dir}/state/certs/slow') + + def upload(): + body = ApplicationTLS().conf(bundle, '/certificates/slow') + on_disk = stored.exists() and stored.read_bytes() == bundle + done['upload'] = (body, on_disk) + + def change(): + start = time.monotonic() + body = ApplicationTLS().conf( + {"http": {"idle_timeout": 77}}, 'settings' + ) + done['change'] = (time.monotonic() - start, body) + + try: + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(upload) + + # The store child is in its first fsync(2) now. + time.sleep(1) + + skip_fds_check(router=True) + skip_alert(fr'process {router[0]} exited on signal 9') + subprocess.call(['kill', '-9', router[0]]) + + for _ in range(50): + new = children('unit: router') + if new and new != router: + break + time.sleep(0.1) + + # Let the new router get the configuration. + time.sleep(0.5) + + second = pool.submit(change) + + first.result(timeout=30) + second.result(timeout=30) + + finally: + tracer.terminate() + tracer.wait(10) + + assert 'success' in done['upload'][0], 'one 200 for the upload' + assert done['upload'][1], 'the bundle is on disk at the 200' + assert 'success' in done['change'][1], 'the change is applied' + + # The store runs about 4 s, and it started about 1.5 s before the + # change was sent. A change that did not wait takes milliseconds. + assert done['change'][0] > 1, 'the change waited for the store' + + # Main has answered the upload; the controller must still run. + time.sleep(0.5) + + assert children('unit: controller') == controller, 'controller survived' + assert client.conf_get('settings/http/idle_timeout') == 77 + + +def test_tls_certificate_update_router_restart( + skip_alert, skip_fds_check, unit_pid +): + """A listener uses the bundle, and the router restarts while main + stores it. The store ends while the new router still applies its + first configuration, because the application takes 8 s to start. The + controller sends the configuration again only after that apply ends. + Before, it sent it at once. The two applies ran at the same time in + the router: the upload got 500, the listener stopped answering, and the + next change crashed the router. + + strace delays each fsync(2) of main and its children by 2 s, so the + store child of the upload runs for about 4 s.""" + + def children(name): + return unit_children(unit_pid, name) + + bundles = {} + + for name in ('old', 'new'): + client.certificate(name, False) + + with open(f'{option.temp_dir}/{name}.key', 'rb') as k, open( + f'{option.temp_dir}/{name}.crt', 'rb' + ) as c: + bundles[name] = k.read() + c.read() + + def served(): + pem = ssl.get_server_certificate(('127.0.0.1', 8080), timeout=10) + return ssl.PEM_cert_to_DER_cert(pem) + + def der(name): + pem = Path(f'{option.temp_dir}/{name}.crt').read_text(encoding='utf-8') + return ssl.PEM_cert_to_DER_cert(pem) + + assert 'success' in client.conf(bundles['old'], '/certificates/c') + + app = f'{option.test_dir}/python/slow_start' + + assert 'success' in client.conf( + { + "listeners": { + "*:8080": { + "pass": "applications/slow_start", + "tls": {"certificate": "c"}, + } + }, + "applications": { + "slow_start": { + "type": "python", + "processes": 1, + "path": app, + "working_directory": app, + "module": "wsgi", + "environment": {"UNIT_SLOW_START": "8"}, + } + }, + } + ) + + assert served() == der('old'), 'the old certificate before' + + router = children('unit: router') + assert len(router) == 1, 'one router' + + stored = Path(f'{option.temp_dir}/state/certs/c') + + tracer = trace_fsync(unit_pid) + + done = {} + + def upload(): + start = time.monotonic() + body = ApplicationTLS().conf(bundles['new'], '/certificates/c') + done['upload'] = (time.monotonic() - start, body) + + try: + with ThreadPoolExecutor(max_workers=1) as pool: + first = pool.submit(upload) + + # The store child is in its first fsync(2) now. + time.sleep(1) + + assert stored.read_bytes() == bundles['old'], 'store not done' + + skip_fds_check(router=True) + skip_alert(fr'process {router[0]} exited on signal 9') + subprocess.call(['kill', '-9', router[0]]) + + first.result(timeout=60) + + finally: + tracer.terminate() + tracer.wait(10) + + elapsed, body = done['upload'] + + assert body.get('success') == 'Certificate chain updated.', 'applied' + + # The store ends after about 4 s. The first configuration of the new + # router ends after the application starts, 8 s after the kill. + assert elapsed > 6, 'the upload waited for the first configuration' + + assert served() == der('new'), 'the new certificate' + + router = children('unit: router') + assert len(router) == 1, 'one new router' + + assert 'success' in client.conf( + {"http": {"idle_timeout": 77}}, 'settings' + ), 'a later change' + + assert served() == der('new'), 'the new certificate after the change' + assert client.get_ssl()['status'] == 200, 'the listener answers' + assert children('unit: router') == router, 'the router survived' + + +def test_tls_certificate_update_controller_restart( + skip_alert, skip_fds_check, unit_pid +): + """A listener uses the bundle, and the controller exits while main + stores a new one. Main starts the new controller only when the store + ends, so the new controller reads the new bundle from disk. It lists + the new certificate, and the router serves it. When main started the + controller at once, the new controller read the old bundle. The answer + of the store went to the controller that had gone, so the router served + the old certificate after the store ended. + + strace delays each fsync(2) of main and its children by 2 s, so the + store child of the upload runs for about 4 s.""" + + def children(name): + return unit_children(unit_pid, name) + + bundles = {} + + for name in ('old', 'new'): + client.certificate(name, False) + + with open(f'{option.temp_dir}/{name}.key', 'rb') as k, open( + f'{option.temp_dir}/{name}.crt', 'rb' + ) as c: + bundles[name] = k.read() + c.read() + + def served(): + pem = ssl.get_server_certificate(('127.0.0.1', 8080), timeout=10) + return ssl.PEM_cert_to_DER_cert(pem) + + def der(name): + pem = Path(f'{option.temp_dir}/{name}.crt').read_text(encoding='utf-8') + return ssl.PEM_cert_to_DER_cert(pem) + + client.load('empty') + + assert 'success' in client.conf(bundles['old'], '/certificates/c') + + add_tls(cert='c') + + assert served() == der('old'), 'the old certificate before' + + controller = children('unit: controller') + assert len(controller) == 1, 'one controller' + + stored = Path(f'{option.temp_dir}/state/certs/c') + + tracer = trace_fsync(unit_pid) + + done = {} + + def upload(): + done['upload'] = ApplicationTLS().put( + url='/certificates/c', + sock_type='unix', + addr=f'{option.temp_dir}/control.unit.sock', + body=bundles['new'], + ) + + try: + with ThreadPoolExecutor(max_workers=1) as pool: + first = pool.submit(upload) + + # The store child is in its first fsync(2) now. + time.sleep(1) + + assert stored.read_bytes() == bundles['old'], 'store not done' + + skip_fds_check(controller=True) + skip_alert(fr'process {controller[0]} exited on signal 9') + subprocess.call(['kill', '-9', controller[0]]) + + first.result(timeout=30) + + # The store child is a fork of main, with the title of main. + for _ in range(100): + if not children('unit: main'): + break + time.sleep(0.1) + + assert not children('unit: main'), 'the store ended' + + for _ in range(100): + new = children('unit: controller') + if new and new != controller: + break + time.sleep(0.1) + + assert new and new != controller, 'a new controller' + + finally: + tracer.terminate() + tracer.wait(10) + + assert done['upload'] == {}, 'the controller exited before it answered' + assert stored.read_bytes() == bundles['new'], 'the new bundle on disk' + + # The new controller listens after the router applies its configuration. + assert ( + client.conf_get('/certificates/c/chain/0/subject/common_name') + == 'new' + ), 'the controller lists the new certificate' + + assert served() == der('new'), 'the router serves the new certificate' + assert client.get_ssl()['status'] == 200, 'the listener answers' + + def test_tls_certificate_update_long_name(): client.certificate('default', False) From 6ccfa2a623a4775146d292fb6b3c5c7a2107d784 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 12:35:56 +0200 Subject: [PATCH 358/391] http: serve a static .svgz file as gzip-coded image/svg+xml A .svgz file is an SVG image stored with gzip coding. The built-in MIME table had no entry for it, so the static handler sent no Content-Type and no Content-Encoding, and a browser did not show the image. With compression configured and no "types" list, the router also compressed the file a second time and labelled the result with one "gzip". Map ".svgz" to image/svg+xml, and add "Content-Encoding: gzip" when the extension is ".svgz" and it resolves to image/svg+xml. The field is added before nxt_http_comp_check_acceptable(), which skips a response that already has a Content-Encoding. So the stored bytes go out unchanged, the ETag stays strong, no Vary is added, and a Range applies to the stored bytes. A 206 and a HEAD carry the same field. A 304 and a 416 do not, because they have no body. The ETag of a .svgz file ends in "-gzip", for example "6ac245de-58-gzip". Before this change the file had the tag "6ac245de-58". A cache stored that tag with a response that had no coding, or with a second gzip under W/. When the cache revalidates, the old tag no longer matches. So it gets a full 200 with the new header fields, not a 304 that keeps the broken entry. If-Match and If-Range compare against the new tag in the same way. The Last-Modified date did not change, so a date cannot tell the old response from the new one. For a .svgz file, If-Modified-Since does not give a 304, and a date in If-Range does not give a 206. The client gets the full 200. If-Unmodified-Since is not changed. A 412 makes no cache reuse an entry, and a request that passes gets the full 200. The extension matches without regard to case, as the type lookup does. An operator who maps ".svgz" to another type in "mime_types" gets that type and no coding. The coding is not negotiated. A client without gzip in Accept-Encoding still gets the gzip bytes, because no other representation exists. Apache does the same with "AddEncoding gzip svgz". A share "types" rule that names image/svg+xml or image/* now matches a .svgz file; before, the file had no type and such a rule refused it. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 Reviewed-by: Codex Reviewed-by: DeepSeek --- CHANGES | 19 +++ docs/changes.xml | 23 ++++ src/nxt_http_static.c | 144 ++++++++++++++++++---- test/test_static.py | 208 ++++++++++++++++++++++++++++++++ test/test_static_compression.py | 55 +++++++++ test/test_static_types.py | 18 +++ 6 files changed, 444 insertions(+), 23 deletions(-) diff --git a/CHANGES b/CHANGES index a928fd389..de2cc9fbd 100644 --- a/CHANGES +++ b/CHANGES @@ -283,6 +283,25 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 such a response gave 502 when it carried chunked together with Content-Length. + *) Change: ".svgz" added to the default MIME type list as + "image/svg+xml". A share "types" rule that names image/svg+xml or + image/* now matches a .svgz file; before, the file had no type and + such a rule refused it. + + *) Feature: a static ".svgz" file is served as "image/svg+xml" with + "Content-Encoding: gzip". Before, it had no Content-Type, and a + browser did not show it. The stored gzip bytes are sent + unchanged: compression does not code them again, the ETag stays + strong, and a Range applies to the stored bytes. A client without + gzip in Accept-Encoding also gets the gzip bytes, as with + "AddEncoding gzip svgz" in Apache. A "mime_types" setting that + maps ".svgz" to another type adds no coding. The ETag of a .svgz + file ends in "-gzip", so a cache that stored the old response + gets a full response, not a 304. A .svgz file never answers 304 + to If-Modified-Since, and a date in If-Range never resumes it, so + a client that revalidates by date only downloads the file again + each time. + *) Feature: the "body_min_rate" and "send_min_rate" options in "settings/http" set a minimum client rate in bytes per second on HTTP/1 connections. The body_read_timeout and send_timeout timers diff --git a/docs/changes.xml b/docs/changes.xml index 592210f1f..e8b89b85b 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -410,6 +410,29 @@ it carried chunked together with Content-Length. + + +".svgz" added to the default MIME type list as "image/svg+xml". A share "types" +rule that names image/svg+xml or image/* now matches a .svgz file; before, the +file had no type and such a rule refused it. + + + + + +a static ".svgz" file is served as "image/svg+xml" with "Content-Encoding: +gzip". Before, it had no Content-Type, and a browser did not show it. The +stored gzip bytes are sent unchanged: compression does not code them again, the +ETag stays strong, and a Range applies to the stored bytes. A client without +gzip in Accept-Encoding also gets the gzip bytes, as with "AddEncoding gzip +svgz" in Apache. A "mime_types" setting that maps ".svgz" to another type adds +no coding. The ETag of a .svgz file ends in "-gzip", so a cache that stored the +old response gets a full response, not a 304. A .svgz file never answers 304 to +If-Modified-Since, and a date in If-Range never resumes it, so a client that +revalidates by date only downloads the file again each time. + + + the "body_min_rate" and "send_min_rate" options in "settings/http" set diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 2a8f19fa3..bc8ec11e1 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -168,12 +168,12 @@ static void nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, static void nxt_http_static_next(nxt_task_t *task, nxt_http_request_t *r, nxt_http_static_ctx_t *ctx, nxt_http_status_t status); static nxt_http_status_t nxt_http_static_preconditions(nxt_http_request_t *r, - nxt_str_t *etag, nxt_bool_t weak, nxt_time_t mtime); + nxt_str_t *etag, nxt_bool_t weak, nxt_time_t mtime, nxt_bool_t date_ok); static nxt_bool_t nxt_http_static_etag_match(nxt_str_t *list, nxt_str_t *etag, nxt_bool_t own_weak, nxt_bool_t strong); static nxt_http_status_t nxt_http_static_range(nxt_http_request_t *r, - nxt_str_t *etag, nxt_bool_t weak, nxt_time_t mtime, nxt_off_t size, - nxt_off_t *start, nxt_off_t *end); + nxt_str_t *etag, nxt_bool_t weak, nxt_time_t mtime, nxt_bool_t date_ok, + nxt_off_t size, nxt_off_t *start, nxt_off_t *end); #if (NXT_HAVE_OPENAT2) static u_char *nxt_http_static_chroot_match(u_char *chr, u_char *shr); #endif @@ -497,21 +497,25 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, nxt_file_t *f, file; nxt_file_info_t fi; nxt_off_t range_start, range_end; - nxt_bool_t is_range, etag_weak; + nxt_bool_t is_range, etag_weak, svgz; nxt_http_status_t rstatus; - nxt_http_field_t *field; + nxt_http_field_t *field, *coding; nxt_http_status_t status, pcond; nxt_router_conf_t *rtcf; nxt_http_action_t *action; nxt_work_handler_t body_handler; nxt_http_static_conf_t *conf; + static const nxt_str_t svgz_exten = nxt_string(".svgz"); + static const nxt_str_t svg_mtype = nxt_string("image/svg+xml"); + action = ctx->action; conf = action->u.conf; rtcf = r->conf->socket_conf->router_conf; f = NULL; mtype = NULL; + coding = NULL; shr = &ctx->share; index = &conf->index; @@ -716,6 +720,30 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, field->value = p; field->value_length = nxt_http_date(p, &tm) - p; + if (exten.start == NULL) { + nxt_http_static_extract_extension(shr, &exten); + } + + if (mtype == NULL) { + mtype = nxt_http_static_mtype_get(&rtcf->mtypes_hash, &exten); + } + + /* + * A ".svgz" file is an SVG image stored with gzip coding, so it is + * sent with "Content-Encoding: gzip", as Apache does with + * "AddEncoding gzip svgz". The coding is not negotiated. A client + * without gzip in Accept-Encoding still gets the gzip bytes, because + * no other representation exists. + * + * The extension must also resolve to image/svg+xml. An operator who + * maps ".svgz" to another type in "mime_types" gets no coding. + * + * The decision is made here because it also changes the ETag below. + */ + + svgz = (nxt_strcasestr_eq(&exten, &svgz_exten) + && nxt_strcasestr_eq(mtype, &svg_mtype)); + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); if (nxt_slow_path(field == NULL)) { goto fail; @@ -753,7 +781,19 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, etag_weak = ((nxt_time_t) nxt_thread_time(task->thread) <= (nxt_time_t) nxt_file_mtime(&fi)); - length = nxt_length("W/") + NXT_TIME_T_HEXLEN + NXT_OFF_T_HEXLEN + 3; + /* + * The tag of a ".svgz" file ends in "-gzip". Before the file was + * sent with gzip coding, its tag had no suffix. The response had no + * coding, or it was compressed a second time under the weak form of + * the tag. A cache that stored either response sends that tag back + * in If-None-Match. The new tag does not match it, so the cache + * gets a full 200 with the new header fields instead of a 304 that + * keeps the old entry. Apache's mod_deflate adds the same suffix + * to the tag of a response it compresses. + */ + + length = nxt_length("W/") + NXT_TIME_T_HEXLEN + NXT_OFF_T_HEXLEN + + nxt_length("-gzip") + 3; p = nxt_mp_nget(r->mem_pool, length); if (nxt_slow_path(p == NULL)) { @@ -774,9 +814,11 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, */ end = field->value + length; - field->value_length = nxt_sprintf(p, end, "\"%xT-%xO\"", + field->value_length = nxt_sprintf(p, end, "\"%xT-%xO%s\"", (nxt_time_t) nxt_file_mtime(&fi), - nxt_file_size(&fi)) + nxt_file_size(&fi), + svgz ? (u_char *) "-gzip" + : (u_char *) "") - field->value; /* @@ -789,14 +831,6 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, etag.start = p; etag.length = field->value_length - (p - field->value); - if (exten.start == NULL) { - nxt_http_static_extract_extension(shr, &exten); - } - - if (mtype == NULL) { - mtype = nxt_http_static_mtype_get(&rtcf->mtypes_hash, &exten); - } - if (mtype->length != 0) { field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); if (nxt_slow_path(field == NULL)) { @@ -811,6 +845,28 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, r->resp.mime_type = mtype; + /* + * The Content-Encoding of a ".svgz" file is added before the + * acceptability check below. That check skips a response that + * already has a Content-Encoding. So the file is not compressed a + * second time, its ETag stays strong, and a Range applies to the + * stored bytes. + */ + + if (svgz) { + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); + if (nxt_slow_path(field == NULL)) { + goto fail; + } + + nxt_http_field_name_set(field, "Content-Encoding"); + + field->value = (u_char *) "gzip"; + field->value_length = nxt_length("gzip"); + + coding = field; + } + /* * RFC 9110 Sect. 13.2.1: an ordinary failure outranks a precondition. * If no acceptable representation exists the answer is 406, and it @@ -836,8 +892,18 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } + /* + * A response with the stored coding of a ".svgz" file has a coding + * that an older cached response of the same file may lack. The + * Last-Modified date did not change when the coding was added, so a + * date cannot tell the two apart. Only the ETag can. So a date + * gives no 304 and no range for such a response. A client that + * sends If-None-Match or an entity-tag in If-Range is not affected. + */ + pcond = nxt_http_static_preconditions(r, &etag, etag_weak, - nxt_file_mtime(&fi)); + nxt_file_mtime(&fi), + coding == NULL); if (pcond != NXT_HTTP_OK) { nxt_file_close(task, f); @@ -858,6 +924,17 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, r->status = NXT_HTTP_NOT_MODIFIED; r->resp.content_length_n = -1; + /* + * RFC 9110 Sect. 15.4.5: a 304 should not carry representation + * metadata such as Content-Encoding. A cache keeps the field it + * stored with the 200. The compressed path also sends a 304 + * without it. + */ + + if (coding != NULL) { + coding->skip = 1; + } + body_handler = NULL; goto send; @@ -885,7 +962,7 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, */ rstatus = nxt_http_static_range(r, &etag, etag_weak, - nxt_file_mtime(&fi), + nxt_file_mtime(&fi), coding == NULL, nxt_file_size(&fi), &range_start, &range_end); @@ -910,6 +987,16 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, r->status = NXT_HTTP_RANGE_NOT_SATISFIABLE; r->resp.content_length_n = 0; + /* + * A 416 has no body, so it has no content coding. The + * error responses (406, 412, 500) need no such step: + * nxt_http_request_error() drops every field added here. + */ + + if (coding != NULL) { + coding->skip = 1; + } + field = nxt_http_resp_field_zero_add(&r->resp, r->mem_pool); if (nxt_slow_path(field == NULL)) { goto fail; @@ -1167,11 +1254,17 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, * The later step in each pair is consulted only when the earlier one is * absent: a client that sends both an entity-tag and a date is asking to be * judged by the entity-tag, even when the tag does not match. + * + * "date_ok" is 0 when the date does not identify the representation. Then + * If-Modified-Since never gives a 304, because a 304 tells a cache to reuse + * the entry it has, and that entry can be stale. If-Unmodified-Since needs + * no such gate. A 412 makes no cache reuse an entry. A request that passes + * gets the full 200 with the current header fields. */ static nxt_http_status_t nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, - nxt_bool_t weak, nxt_time_t mtime) + nxt_bool_t weak, nxt_time_t mtime, nxt_bool_t date_ok) { nxt_str_t value; nxt_time_t date; @@ -1306,7 +1399,7 @@ nxt_http_static_preconditions(nxt_http_request_t *r, nxt_str_t *etag, return NXT_HTTP_OK; } - if (ims != NULL) { + if (ims != NULL && date_ok) { date = nxt_time_parse(ims->value, ims->value_length); /* @@ -1495,12 +1588,15 @@ nxt_http_static_range_number(u_char **p, u_char *end) * to serve as a 206; both are clamped to [0, size - 1]. * NXT_HTTP_RANGE_NOT_SATISFIABLE -- the single range is out of bounds; the * caller answers 416 with a "Content-Range: bytes STAR/size" header. + * + * "date_ok" is 0 when the date does not identify the representation. Then + * a date in If-Range never matches. */ static nxt_http_status_t nxt_http_static_range(nxt_http_request_t *r, nxt_str_t *etag, - nxt_bool_t weak, nxt_time_t mtime, nxt_off_t size, nxt_off_t *start, - nxt_off_t *end) + nxt_bool_t weak, nxt_time_t mtime, nxt_bool_t date_ok, nxt_off_t size, + nxt_off_t *start, nxt_off_t *end) { u_char *p, *last; nxt_off_t a, b, suffix; @@ -1576,7 +1672,8 @@ nxt_http_static_range(nxt_http_request_t *r, nxt_str_t *etag, * stale one. Refusing the If-Range costs a full response. */ - match = (!weak && date != (nxt_time_t) -1 && date == mtime); + match = (date_ok && !weak && date != (nxt_time_t) -1 + && date == mtime); } if (!match) { @@ -1970,6 +2067,7 @@ nxt_http_static_mtypes_init(nxt_mp_t *mp, nxt_lvlhsh_t *hash) { nxt_string("text/css"), ".css" }, { nxt_string("image/svg+xml"), ".svg" }, + { nxt_string("image/svg+xml"), ".svgz" }, { nxt_string("image/webp"), ".webp" }, { nxt_string("image/png"), ".png" }, { nxt_string("image/apng"), ".apng" }, diff --git a/test/test_static.py b/test/test_static.py index b4d616c30..b9968754c 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -1,3 +1,4 @@ +import gzip import os import socket import time @@ -910,6 +911,213 @@ def test_static_mime_types_correct(): ), 'mime_types same extensions case insensitive' +SVG = b'' + + +def svgz_write(path): + # mtime=0 keeps the gzip bytes the same on every run. + data = gzip.compress(SVG, mtime=0) + Path(path).write_bytes(data) + age_file(path) + return data + + +def raw_request(method='GET', url='/a.svgz', **headers): + # Raw bytes: the body is gzip data, so it must not be decoded as UTF-8. + raw = client.http( + method, + url=url, + headers={'Host': 'localhost', 'Connection': 'close', **headers}, + encoding='latin-1', + raw_resp=True, + ) + head, _, body = raw.partition('\r\n\r\n') + lines = head.split('\r\n') + status = int(lines[0].split(' ')[1]) + hdrs = dict(line.split(': ', 1) for line in lines[1:]) + return status, hdrs, body.encode('latin-1') + + +def test_static_svgz(temp_dir): + # A ".svgz" file is served as image/svg+xml with "Content-Encoding: gzip", + # and its stored bytes are sent unchanged. Before, it had no Content-Type + # and no Content-Encoding, so a browser did not display it. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + status, headers, body = raw_request() + assert status == 200 + assert headers.get('Content-Type') == 'image/svg+xml' + assert headers.get('Content-Encoding') == 'gzip' + assert headers['Content-Length'] == str(len(data)) + assert body == data, 'the stored bytes' + assert gzip.decompress(body) == SVG + assert 'Vary' not in headers, 'the coding is not negotiated' + + # A client that does not accept gzip gets the same bytes. No other + # representation exists. + status, headers, body = raw_request(**{'Accept-Encoding': 'identity'}) + assert status == 200 + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + # HEAD sends the same header fields and no body. + status, headers, body = raw_request('HEAD') + assert status == 200 + assert headers.get('Content-Type') == 'image/svg+xml' + assert headers.get('Content-Encoding') == 'gzip' + assert headers['Content-Length'] == str(len(data)) + assert body == b'', 'no body on HEAD' + + # A 304 carries no Content-Encoding. + status, headers, body = raw_request(**{'If-None-Match': headers['ETag']}) + assert status == 304 + assert 'Content-Encoding' not in headers + assert body == b'' + + +def test_static_svgz_case_insensitive(temp_dir): + # The extension matches without regard to case, as the type lookup does. + data = svgz_write(f'{temp_dir}/assets/B.SVGZ') + + status, headers, body = raw_request(url='/B.SVGZ') + assert status == 200 + assert headers.get('Content-Type') == 'image/svg+xml' + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + +def test_static_svgz_range(temp_dir): + # A range applies to the stored gzip bytes, and the 206 keeps + # "Content-Encoding: gzip": the slice is part of the gzip data. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + status, headers, body = raw_request(Range='bytes=0-9') + assert status == 206 + assert headers.get('Content-Encoding') == 'gzip' + assert headers['Content-Range'] == f'bytes 0-9/{len(data)}' + assert body == data[:10] + + # The ETag is strong, so a matching If-Range resumes the range. + etag = raw_request('HEAD')[1]['ETag'] + assert not etag.startswith('W/'), 'strong ETag' + + status, headers, body = raw_request( + Range='bytes=10-', **{'If-Range': etag} + ) + assert status == 206 + assert headers.get('Content-Encoding') == 'gzip' + assert body == data[10:] + + +def test_static_svgz_etag(temp_dir): + # The tag of a ".svgz" file has a "-gzip" suffix. Before, the file had + # the plain tag, and a cache stored it with no coding or with a second + # gzip under W/. That tag must no longer match, so the cache gets a full + # 200 with the coding instead of a 304 that keeps the old entry. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + # The same bytes and mtime under another name get the plain tag. + Path(f'{temp_dir}/assets/a.bin').write_bytes(data) + mtime = os.stat(f'{temp_dir}/assets/a.svgz').st_mtime + os.utime(f'{temp_dir}/assets/a.bin', (mtime, mtime)) + + etag = raw_request('HEAD')[1]['ETag'] + old = raw_request('HEAD', url='/a.bin')[1]['ETag'] + assert not old.startswith('W/'), 'strong ETag' + assert etag == f'{old[:-1]}-gzip"', 'the tag has the suffix' + + for tag in (old, f'W/{old}'): + status, headers, body = raw_request(**{'If-None-Match': tag}) + assert status == 200, f'old tag {tag} does not match' + assert headers['ETag'] == etag + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + status, headers, body = raw_request(**{'If-None-Match': etag}) + assert status == 304, 'the new tag matches' + assert 'Content-Encoding' not in headers + assert body == b'' + + assert raw_request(**{'If-Match': old})[0] == 412, 'If-Match old tag' + assert raw_request(**{'If-Match': etag})[0] == 200, 'If-Match new tag' + + # A range resumed against the old tag gets the whole file. + status, headers, body = raw_request(Range='bytes=10-', **{'If-Range': old}) + assert status == 200, 'If-Range old tag' + assert body == data + + +def test_static_svgz_date_validators(temp_dir): + # The Last-Modified date of a ".svgz" file did not change when its + # response gained the coding. So a date must not give a 304 or a range: + # a cache that stored the old response would keep it. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + # Control: the same bytes and mtime under another name. + Path(f'{temp_dir}/assets/a.bin').write_bytes(data) + mtime = os.stat(f'{temp_dir}/assets/a.svgz').st_mtime + os.utime(f'{temp_dir}/assets/a.bin', (mtime, mtime)) + + date = raw_request('HEAD')[1]['Last-Modified'] + later = formatdate(time.time() + 3600, usegmt=True) + + for ims in (date, later): + assert ( + raw_request(url='/a.bin', **{'If-Modified-Since': ims})[0] == 304 + ), f'control If-Modified-Since {ims}' + + status, headers, body = raw_request(**{'If-Modified-Since': ims}) + assert status == 200, f'If-Modified-Since {ims}' + assert headers['ETag'].endswith('-gzip"') + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + status, _, body = raw_request( + url='/a.bin', Range='bytes=10-', **{'If-Range': date} + ) + assert status == 206, 'control If-Range date' + assert body == data[10:] + + status, headers, body = raw_request(Range='bytes=10-', **{'If-Range': date}) + assert status == 200, 'If-Range date' + assert headers.get('Content-Encoding') == 'gzip' + assert body == data + + # If-Unmodified-Since still works: a 412 from a date stays correct. + earlier = formatdate(mtime - 3600, usegmt=True) + assert raw_request(**{'If-Unmodified-Since': earlier})[0] == 412 + assert raw_request(**{'If-Unmodified-Since': date})[0] == 200 + + +def test_static_svgz_range_not_satisfiable(temp_dir): + # A 416 has no body, so it carries no Content-Encoding. + data = svgz_write(f'{temp_dir}/assets/a.svgz') + + status, headers, body = raw_request(Range=f'bytes={len(data)}-') + assert status == 416 + assert headers['Content-Range'] == f'bytes */{len(data)}' + assert 'Content-Encoding' not in headers + assert body == b'' + + +def test_static_svgz_mime_types_remap(temp_dir): + # The coding is added only while ".svgz" resolves to image/svg+xml. An + # operator who maps it to another type gets that type and no coding. + svgz_write(f'{temp_dir}/assets/a.svgz') + + assert raw_request()[1].get('Content-Encoding') == 'gzip' + + assert 'success' in client.conf( + {"text/plain": [".log", "README"], "application/x-blob": ".svgz"}, + 'settings/http/static/mime_types', + ) + + status, headers, _ = raw_request() + assert status == 200 + assert headers.get('Content-Type') == 'application/x-blob' + assert 'Content-Encoding' not in headers + + @pytest.mark.skip('not yet') def test_static_mime_types_invalid(temp_dir): assert 'error' in client.http( diff --git a/test/test_static_compression.py b/test/test_static_compression.py index 62d7dc8b6..34cbfb623 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -1,4 +1,5 @@ import gzip +import os import zlib from pathlib import Path @@ -444,6 +445,60 @@ def test_static_compression_range_identity_refused(temp_dir): assert gzip.decompress(body) == data, 'the whole file, correctly coded' +def test_static_compression_svgz_not_compressed_twice(temp_dir): + # A ".svgz" file already has gzip coding. With every type compressed, + # it must still go out as its stored bytes: one gzip layer, a strong + # ETag, and no Vary, because the coding is not negotiated. Before, it + # had no type, so it was compressed again and labelled with one "gzip". + svg = b'' * 10 + data = gzip.compress(svg, mtime=0) + path = f'{temp_dir}/assets/a.svgz' + Path(path).write_bytes(data) + when = os.stat(path).st_mtime - 2 + os.utime(path, (when, when)) + + assert 'success' in client.conf_delete( + 'settings/http/compression/types' + ), 'compress every type' + + status, headers, body = _raw_get( + url='/a.svgz', **{'Accept-Encoding': 'gzip'} + ) + assert status == 200 + assert headers.get('Content-Type') == 'image/svg+xml' + assert headers.get('Content-Encoding') == 'gzip' + assert body == data, 'the stored bytes, not compressed again' + assert gzip.decompress(body) == svg + assert not headers['ETag'].startswith('W/'), 'strong ETag' + assert 'Vary' not in headers + + # "identity;q=0" does not drop the Range: the file is not sent as + # identity, and the slice is of the gzip bytes. + status, headers, body = _raw_get( + url='/a.svgz', + **{'Accept-Encoding': 'gzip, identity;q=0', 'Range': 'bytes=0-9'}, + ) + assert status == 206 + assert headers.get('Content-Encoding') == 'gzip' + assert headers['Content-Range'] == f'bytes 0-9/{len(data)}' + assert body == data[:10] + + # A client that accepts no coding still gets the gzip bytes, with 200. + # This is intended: no other representation of the file exists. For a + # file that can be sent as identity, the second header gives 406. + for accept in ('identity;q=0', 'identity;q=0, *;q=0'): + status, headers, body = _raw_get( + url='/a.svgz', **{'Accept-Encoding': accept} + ) + assert status == 200, accept + assert headers.get('Content-Encoding') == 'gzip', accept + assert body == data, accept + + assert ( + _raw_get(**{'Accept-Encoding': 'identity;q=0, *;q=0'})[0] == 406 + ), 'big.css: nothing acceptable' + + def test_static_compression_identity_refused_below_min_length(): # "min_length" is 10 and tiny.css is two bytes. So the gzip that made # this request serveable is never applied, and the fallback is the diff --git a/test/test_static_types.py b/test/test_static_types.py index e931d949a..9f2f893ba 100644 --- a/test/test_static_types.py +++ b/test/test_static_types.py @@ -1,3 +1,4 @@ +import gzip from pathlib import Path import pytest @@ -172,3 +173,20 @@ def test_static_types_custom_mime(temp_dir): {"share": f'{temp_dir}/assets$uri', "types": ["test/mime-type"]} ) check_body('/file', '') + + +def test_static_types_svgz(temp_dir): + # ".svgz" is in the default MIME table as image/svg+xml. So a "types" + # rule that names image/* serves it, with its stored gzip coding. + # Before, the file had no type and the rule refused it with 403. + data = gzip.compress(b'', mtime=0) + Path(f'{temp_dir}/assets/a.svgz').write_bytes(data) + + action_update({"share": f'{temp_dir}/assets$uri', "types": ["image/*"]}) + + resp = client.get(url='/a.svgz', encoding='latin-1') + assert resp['status'] == 200, 'image/* serves .svgz' + assert resp['headers']['Content-Type'] == 'image/svg+xml' + assert resp['headers']['Content-Encoding'] == 'gzip' + assert resp['headers']['ETag'].endswith('-gzip"') + assert resp['body'].encode('latin-1') == data From 6c08e4ec5db51c33f3fe8806f5d3eacf3c50913d Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 28 Sep 2026 07:55:10 +0200 Subject: [PATCH 359/391] http: correct two comments about pattern decoding and PCRE2 ovecsize Request paths are decoded by nxt_http_parse_complex_target(), not by nxt_decode_uri(), so the two sides are not symmetric. The parser removes "/./" and "/../" segments and merges repeated slashes. A ".." inside a segment is kept. Arguments and the query are decoded by nxt_http_arguments_parse(), which copies a bad "%XX" as is. Only the controller sets encoded_slashes. PCRE2 treats an ovecsize of 0 as 1, so passing 0 is not undefined. The man page pcre2_match_data_create(3) says so in 10.23 and in 10.46, and pcre2_match_data_create() clamps the value in both. This replaces the sentence that https://github.com/freeunitorg/freeunit/pull/520 added to the same comment. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- src/nxt_http_route.c | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index 01db1ee94..9c1dd5a32 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -1184,13 +1184,31 @@ nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, /* - * Configured pattern strings are decoded once here at compile time, - * while request URIs are decoded by the HTTP parser before they reach - * the matcher. Both paths funnel through the same nxt_decode_uri / - * nxt_decode_uri_plus helpers; %XX semantics are symmetric, so a - * "%2e%2e" in the configured pattern will compare against the same - * bytes a request "%2e%2e" decoded into. If either helper's behaviour - * changes, route patterns and request URIs MUST be updated in lock-step. + * Configured pattern strings are decoded once here, at configuration time, + * by nxt_decode_uri() ("uri") or nxt_decode_uri_plus() ("arguments" and + * "query"). A regex pattern ("~...") is compiled as written and is not + * decoded. + * + * The request side does not use these helpers, and the two sides are not + * symmetric. + * + * The request path is decoded by nxt_http_parse_complex_target() in + * src/nxt_http_parse.c: + * - most decoded bytes go back through its state machine, so an encoded + * dot segment such as "/%2e%2e/" is normalized like "/../"; + * - a decoded '%', '#' or '?' is copied as a plain byte; + * - "%00" is rejected; + * - "/./" and "/../" segments and a trailing "/." or "/.." are removed, + * and repeated slashes are merged; + * - "encoded_slashes" keeps "%25" and "%2F" encoded, but only the + * controller sets it. The router never does. + * So a pattern that spells "%2e%2e" or "//" as a whole path segment can + * never match a normalized path. A ".." inside a segment ("/a..b") is + * kept on both sides and compares as plain bytes. + * + * Request arguments and the query are decoded by nxt_http_arguments_parse() + * in src/nxt_http_request.c: '+' becomes a space, a bad or short "%XX" is + * copied as is, and "%00" is accepted. */ static nxt_int_t nxt_http_route_decode_str(nxt_str_t *str, nxt_http_uri_encoding_t encoding) @@ -2167,8 +2185,11 @@ nxt_http_route_pattern(nxt_http_request_t *r, nxt_http_route_pattern_t *pattern, * One match object serves every pattern tested against this * request. Its match data holds the minimum ovector: one * offset pair, the overall match. The matcher does not read - * captures. pcre2_match_data_create() with 0 is undefined in - * the public docs; 1 is the documented minimum. + * captures. pcre2_match_data_create() treats an ovecsize of 0 + * as 1 (pcre2_match_data_create(3)); 1 is passed to make the + * minimum explicit. With PCRE1 the size counts ints, not + * pairs; pcre_exec() then returns 0 on a match, and + * nxt_regex_match() counts 0 as a match. */ r->regex_match = nxt_regex_match_create(r->mem_pool, 1); if (nxt_slow_path(r->regex_match == NULL)) { From d6117108d02d8e693be88fbb0205b4cab5eed487 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 30 Sep 2026 03:19:39 +0200 Subject: [PATCH 360/391] php: free the targets on every error return of nxt_php_start nxt_php_start() allocates the nxt_php_targets array and fills one entry per target. Four error returns did not free them: - nxt_php_set_target() fails for target n of "targets": targets 0 to n-1 and the array stay allocated. - nxt_php_set_target() fails without "targets": the array stays allocated. - nxt_unit_default_init() or nxt_unit_init() fails: all targets and the array stay allocated. These returns now go through one label that calls nxt_php_cleanup_targets(). The array comes from nxt_zalloc(), so a target that was never set has NULL fields, and the cleanup skips them. nxt_php_set_target() sets the fields that it frees on error to NULL, so the cleanup does not free them twice. The cleanup also frees the array and resets nxt_php_targets and nxt_php_targets_count. The worker exits right after these errors, so the memory is held only until exit. The static nxt_php_targets still points to it, so LeakSanitizer does not report it. The change makes the error path free the same memory as the normal exit path. It does not free the PHP state that php_module_startup() creates in nxt_php_setup(); neither path frees that. Co-Authored-By: Claude Opus 5.5 --- src/nxt_php_sapi.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 9ef5b5af5..c59c6a3fc 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -506,28 +506,28 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) ret = nxt_php_set_target(task, &nxt_php_targets[n], value); if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; + goto fail; } } } else { ret = nxt_php_set_target(task, &nxt_php_targets[0], conf->self); if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; + goto fail; } } ret = nxt_unit_default_init(task, &php_init, conf); if (nxt_slow_path(ret != NXT_OK)) { nxt_alert(task, "nxt_unit_default_init() failed"); - return ret; + goto fail; } php_init.callbacks.request_handler = nxt_php_request_handler; unit_ctx = nxt_unit_init(&php_init); if (nxt_slow_path(unit_ctx == NULL)) { - return NXT_ERROR; + goto fail; } nxt_php_unit_ctx = unit_ctx; @@ -541,6 +541,12 @@ nxt_php_start(nxt_task_t *task, nxt_process_data_t *data) exit(0); return NXT_OK; + +fail: + + nxt_php_cleanup_targets(); + + return NXT_ERROR; } From 96d08354afe1b8c560ce69ce83bd018193bdfabf Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 30 Sep 2026 03:22:57 +0200 Subject: [PATCH 361/391] php: require a path separator after the root in the script check nxt_php_set_target() checked that "script" is under "root" with nxt_str_start(), a byte prefix test on the two realpath() results. With "root": "/srv/app", a script that resolves to "/srv/app2/x.php" passed the check. The application served it, with SCRIPT_NAME set to "2/x.php". The test has been a byte prefix test since 0.4. The new nxt_php_path_is_under() requires the root, then '/', then at least one more byte. So a script that resolves to the root directory itself is now also rejected. Before, it was served as an empty 200 response. Both paths come from realpath(), so they have no "." or ".." parts, no symbolic links, and no repeated or trailing '/'. The root "/" is trimmed to length 0. Every absolute path except "/" is under it. In both rejected cases the application fails to start, as before for a script outside the root: the log has the alert "script is not under php root", and requests get 503. The tests cover a script in a sibling directory whose name starts with the root name, a script that is the root, a script in a subdirectory of the root, and the root "/" with an absolute script. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 7 ++++ docs/changes.xml | 10 +++++ src/nxt_php_sapi.c | 28 +++++++++++-- test/test_php_application.py | 78 ++++++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index de2cc9fbd..6a8031472 100644 --- a/CHANGES +++ b/CHANGES @@ -925,6 +925,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 memory, read uninitialized host memory, or corrupt the GC heap. Building the two modules now needs Rust 1.95.0 or newer. + *) Bugfix: the PHP module checked that "script" is under "root" by + comparing the first bytes of the two real paths. With "root": + "/srv/app", a script in "/srv/app2" passed the check and was served. + The script must now be in "root" or in a directory below it. A + "script" that resolves to "root" itself is also rejected; it was + served as an empty 200 response. The bug had appeared in 0.4. + Changes with FreeUnit 1.36.1 28 Aug 2026 *) Feature: configuration validation errors now say where they are. A diff --git a/docs/changes.xml b/docs/changes.xml index e8b89b85b..afdd926f4 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -1241,6 +1241,16 @@ Building the two modules now needs Rust 1.95.0 or newer. + + +the PHP module checked that "script" is under "root" by comparing the first +bytes of the two real paths. With "root": "/srv/app", a script in "/srv/app2" +passed the check and was served. The script must now be in "root" or in a +directory below it. A "script" that resolves to "root" itself is also +rejected; it was served as an empty 200 response. The bug had appeared in 0.4. + + + diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index c59c6a3fc..557cf78dc 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -104,6 +104,8 @@ static void nxt_php_disable(nxt_task_t *task, const char *type, static nxt_int_t nxt_php_dirname(const nxt_str_t *file, nxt_str_t *dir); static void nxt_php_str_trim_trail(nxt_str_t *str, u_char t); static void nxt_php_str_trim_lead(nxt_str_t *str, u_char t); +static nxt_bool_t nxt_php_path_is_under(const nxt_str_t *root, + const nxt_str_t *path); nxt_inline u_char *nxt_realpath(const void *c); static nxt_int_t nxt_php_do_301(nxt_unit_request_info_t *req); @@ -626,9 +628,7 @@ nxt_php_set_target(nxt_task_t *task, nxt_php_target_t *target, target->script_filename.length = nxt_strlen(p); target->script_filename.start = p; - if (!nxt_str_start(&target->script_filename, - target->root.start, target->root.length)) - { + if (!nxt_php_path_is_under(&target->root, &target->script_filename)) { nxt_alert(task, "script is not under php root"); nxt_free(p); nxt_str_null(&target->script_filename); @@ -1014,6 +1014,28 @@ nxt_php_str_trim_lead(nxt_str_t *str, u_char t) } +/* + * Checks that "path" names something below "root": "root", then '/', then + * at least one more byte. A plain prefix test would accept "/srv/app2/x.php" + * for the root "/srv/app". It would also accept the root itself. + * + * Both paths come from realpath(), so they are absolute and have no "." + * or ".." components, no symbolic links and no repeated or trailing '/'. + * nxt_php_str_trim_trail() trims the root "/" to length 0. Every absolute + * path except "/" itself is below that root. + */ + +static nxt_bool_t +nxt_php_path_is_under(const nxt_str_t *root, const nxt_str_t *path) +{ + return path->length > root->length + && path->length - root->length >= 2 + && path->start[root->length] == '/' + && (root->length == 0 + || memcmp(path->start, root->start, root->length) == 0); +} + + nxt_inline u_char * nxt_realpath(const void *c) { diff --git a/test/test_php_application.py b/test/test_php_application.py index 935ba95ed..5966f02a4 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -808,6 +808,84 @@ def test_php_application_script(): assert resp['body'] != '', 'body not empty' +def php_script_conf(root, script): + return client.conf( + { + "listeners": {"*:8080": {"pass": "applications/script"}}, + "applications": { + "script": { + "type": client.get_application_type(), + "processes": {"spare": 0}, + "root": root, + "script": script, + } + }, + } + ) + + +def php_script_tree(temp_dir): + Path(f'{temp_dir}/app/sub').mkdir(parents=True) + Path(f'{temp_dir}/app2').mkdir() + + for path in ('app/sub', 'app2'): + Path(f'{temp_dir}/{path}/index.php').write_text( + '', encoding='utf-8' + ) + + +def test_php_application_script_under_root(temp_dir): + php_script_tree(temp_dir) + + assert 'success' in php_script_conf(f'{temp_dir}/app', 'sub/index.php') + + resp = client.get() + assert resp['status'] == 200, 'script in a subdirectory of the root' + assert resp['body'] == '/sub/index.php', 'SCRIPT_NAME' + + # The root "/" is trimmed to an empty string, and every absolute path is + # under it. The script is appended to the root, so it can be absolute. + script = os.path.realpath(f'{temp_dir}/app/sub/index.php') + + assert 'success' in php_script_conf('/', script) + + resp = client.get() + assert resp['status'] == 200, 'script under the root "/"' + assert resp['body'] == script, 'SCRIPT_NAME under the root "/"' + + +def test_php_application_script_outside_root( + temp_dir, skip_alert, wait_for_record +): + skip_alert(r'script is not under php root') + php_script_tree(temp_dir) + + # "/app2" starts with the bytes of "/app", but it is + # a sibling directory, not a subdirectory. + assert 'success' in php_script_conf( + f'{temp_dir}/app', '../app2/index.php' + ) + + assert client.get()['status'] == 503, 'script in a sibling directory' + assert ( + wait_for_record(r'script is not under php root') is not None + ), 'sibling directory alert' + + +def test_php_application_script_is_root( + temp_dir, skip_alert, wait_for_record +): + skip_alert(r'script is not under php root') + php_script_tree(temp_dir) + + assert 'success' in php_script_conf(f'{temp_dir}/app', '.') + + assert client.get()['status'] == 503, 'script is the root directory' + assert ( + wait_for_record(r'script is not under php root') is not None + ), 'root directory alert' + + def test_php_application_index_default(): assert 'success' in client.conf( { From bb3e54e348bb33970ea68eda5211f0103457d84c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 30 Sep 2026 04:27:34 +0200 Subject: [PATCH 362/391] script: load every stored module after one that fails At startup, nxt_script_info_init() walks the modules that nxt_script_store_load() opened. It moved to the next module only after a module compiled. After a module that did not compile, every later iteration read that same module again. So one broken file in the scripts directory hid every module that readdir(3) returned after it. The loop now indexes the array with its counter. test_njs_modules_load_after_broken stores eight valid modules and one that does not compile, and restarts unitd. Before this change the module after the broken one was missing: "['good1', ..., 'good7'] == ['good0', 'good1', ..., 'good7']". The new test needs --restart, so .github/workflows/build-test.yml now runs test/test_njs_modules.py in the step that passes --restart. Without that the test is skipped in CI. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 3 ++ CHANGES | 5 +++ docs/changes.xml | 8 +++++ src/nxt_script.c | 6 ++-- test/test_njs_modules.py | 62 ++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ed028c334..2c30c4632 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -963,6 +963,8 @@ jobs: # shutdown event, test_tls_certificate_update_restart restarts it to # check that a replaced bundle survives, and test_state_store.py runs # unitd on its own state directory and stops it while a store runs. + # test_njs_modules.py stores njs modules and starts unitd again to + # check what it loads from the state directory. # All need --restart, which the run above does not pass, so they skip # there. Run them again. - name: Run the tests that restart unitd, with --restart @@ -970,6 +972,7 @@ jobs: export UNIT_PYTHONHOME="${UNIT_PYTHON_PREFIX}" sudo -E /usr/bin/pytest-3 --print-log --restart \ test/test_asgi_lifespan.py \ + test/test_njs_modules.py \ test/test_state_store.py \ test/test_tls.py::test_tls_certificate_update_restart if: ${{ !cancelled() && steps.metadata.outputs.module == 'python' }} diff --git a/CHANGES b/CHANGES index 6a8031472..a79b22b4f 100644 --- a/CHANGES +++ b/CHANGES @@ -131,6 +131,11 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 The module now refuses a 64-bit memory. It also reads the base address again after the guest runs. + *) Bugfix: at startup, a stored njs module that did not compile hid + every stored module that was read after it. Those modules were + missing from /js_modules and could not be used. Now only the module + that does not compile is left out. + *) Bugfix: a keep-alive TLS connection that was idle across a reconfiguration could use the TLS settings of the old configuration after they were freed, when it was closed. The connection now keeps diff --git a/docs/changes.xml b/docs/changes.xml index afdd926f4..36dea3238 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -175,6 +175,14 @@ is moved into its cgroup. + + +at startup, a stored njs module that did not compile hid every stored module +that was read after it. Those modules were missing from /js_modules and could +not be used. Now only the module that does not compile is left out. + + + when unitd ran as root and the control socket was given to another user or diff --git a/src/nxt_script.c b/src/nxt_script.c index 0b36f289a..fda3ce567 100644 --- a/src/nxt_script.c +++ b/src/nxt_script.c @@ -159,17 +159,15 @@ nxt_script_info_init(nxt_task_t *task, nxt_array_t *scripts) item = scripts->elts; for (i = 0; i < scripts->nelts; i++) { - script = nxt_script_get(task, &item->name, item->fd); + script = nxt_script_get(task, &item[i].name, item[i].fd); if (nxt_slow_path(script == NULL)) { continue; } - (void) nxt_script_info_save(&item->name, script); + (void) nxt_script_info_save(&item[i].name, script); nxt_script_destroy(script); - - item++; } } diff --git a/test/test_njs_modules.py b/test/test_njs_modules.py index 79e113054..c322c5f88 100644 --- a/test/test_njs_modules.py +++ b/test/test_njs_modules.py @@ -1,3 +1,9 @@ +import os +import shutil +import tempfile +from pathlib import Path + +from conftest import unit_run, unit_stop from unit.applications.proto import ApplicationProto from unit.option import option @@ -125,3 +131,59 @@ def test_njs_settings_js_module_nul(): resp = client.conf({"js_module": ["next\0x"]}, 'settings') assert 'null character' in resp.get('detail', ''), 'array nul' + + +SHORT_MODULE = b"""export default { + "route": function() {return 'next'} +} +""" + + +def start_on(statedir): + # main writes here as root; the directory only has to be traversable. + os.chmod(statedir, 0o755) + unit_run(state_dir=str(statedir)) + + +def test_njs_modules_load_after_broken(requires_restart, skip_alert): + """One stored module that does not compile must not hide the modules + read after it at startup.""" + + skip_alert(r'.*JS compile module.*failed.*') + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + scripts = statedir / 'scripts' + + try: + scripts.mkdir() + + good = [f'good{i}' for i in range(8)] + + for name in good: + (scripts / name).write_bytes(SHORT_MODULE) + + # Pick a name that readdir() returns before at least one good module; + # startup reads the directory in that order. + for i in range(32): + broken = f'broken{i}' + (scripts / broken).write_bytes(b'export default {') + + if os.listdir(scripts)[-1] != broken: + break + + (scripts / broken).unlink() + else: + assert False, 'no name is read before a good module' + + start_on(statedir) + + loaded = client.conf_get('/js_modules') + + assert sorted(n for n in loaded if n in good) == good, 'all loaded' + assert broken not in loaded + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) From 8262f0622be24185bff58eb3c53ef1682830ebb1 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 30 Sep 2026 04:36:01 +0200 Subject: [PATCH 363/391] script: store an njs module in the state store child The controller stored an uploaded njs module in two steps. Main opened "scripts/" with O_RDWR|O_CREAT and no O_TRUNC, and sent the descriptor to the controller. The controller wrote the module into it with one write(2), ignored the result and answered 200. Nothing truncated the file, flushed it or renamed it. So when a longer file of that name was already on disk, the old tail stayed after the new module. Such a file is left, for example, by a write that failed or was cut short, because at startup the controller skips a module that does not compile. The controller validated the upload, but the router loads a module from the file. So the configuration that used the module failed: "JS compile module(m) failed: SyntaxError: Unterminated string". The module store now works as the certificate store does: - The controller copies the module into shared memory and sends NXT_PORT_MSG_SCRIPT_STORE with the name and the size. - nxt_script_store_put_handler() in main checks the sender by its credentials, the name and the size. It reads the module with pread(2) and submits a PUT job to the store child. The child writes "scripts/.store.tmp", flushes it and renames it over "scripts/". No fsync(2) runs in the event loop of main. - The controller answers 200 only when main answers RPC_READY. On an error it answers 500 and deletes the module's metadata, which it used to keep. nxt_controller_store_in_flight() now also reports a module store, so every other change waits, and a GET of the modules or of "/" waits too. As for a certificate, the request of the store is not in the queue of waiting requests. A restarted router makes the controller run that queue again, and a store request in it would run a second time. - A DELETE is a job too. Jobs run in the order they come, so a delete runs after an earlier store of the same module. The controller does not wait for the delete, as for a certificate. It sends no DELETE while a store waits for its answer. Main now reads only the name up to a NUL inside the message. A name must not be empty, longer than 255 bytes, contain "/" or start with ".". The controller answers such a name with 400. A module larger than NXT_SCRIPT_STORE_MAX_SIZE (16 MiB) is answered with 413. Only the router reads a module now. nxt_script_store_get_handler() opens it read-only and never creates it. It also checked rt->certs instead of rt->scripts. At startup, the loader skips every name that starts with ".", so a temporary file of a store that did not finish is not loaded as a module. Tests, each run on the code before this change: - test_njs_modules_replace_torn_file puts half of a 4 KiB module in "scripts/m", restarts unitd and uploads a 60-byte module as "m". Before: "AssertionError: stored 2096 bytes". - test_njs_modules_name_dot uploads ".x" and ".store.tmp". Before: the upload of ".x" succeeded. - test_njs_modules_skip_store_tmp restarts unitd with a valid "scripts/.store.tmp". Before: /js_modules listed ".store.tmp". - test_njs_modules_replace_restart uploads a long module, deletes it, uploads a short one and restarts unitd. It passes before and after. It checks that a delete and a store of the same name run in order. - test_njs_modules_store_router_restart holds the store child with ptrace(2), kills the router and sends a configuration while the store waits. unitd has no test hooks, so ptrace holds the child. The test skips when ptrace is not allowed. With the request of the store in the queue: "{'module': {'error': 'JS module already exists.'}, ..., 'conf': {'success': 'Reconfiguration done.'}}" before the store ended, and the controller then exited on signal 11. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 14 ++ docs/changes.xml | 16 ++ src/nxt_controller.c | 155 +++++++++++++----- src/nxt_main_process.c | 15 +- src/nxt_port.h | 7 +- src/nxt_script.c | 320 +++++++++++++++++++++++++++++++++--- src/nxt_script.h | 14 ++ test/test_njs_modules.py | 343 ++++++++++++++++++++++++++++++++++++++- 8 files changed, 807 insertions(+), 77 deletions(-) diff --git a/CHANGES b/CHANGES index a79b22b4f..f0505f5f5 100644 --- a/CHANGES +++ b/CHANGES @@ -131,6 +131,20 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 The module now refuses a 64-bit memory. It also reads the base address again after the guest runs. + *) Bugfix: an uploaded njs module was written into its file without + truncating it, and without fsync(2). When a longer file of that + name was already on disk, for example one that a failed write had + left, the old tail stayed after the new module. The router read + that file and the configuration failed. The upload was also + answered with 200 when the write failed. Now the main process + stores the module through a temporary file, flushes it and renames + it, in its store child, and the upload is answered only when the + module is on disk. A deletion also runs in the store child. A + module name can no longer start with ".", and a module larger than + 16 MiB is answered with 413. Earlier versions accepted a name that + starts with "."; such a module is no longer loaded or listed after + the upgrade. + *) Bugfix: at startup, a stored njs module that did not compile hid every stored module that was read after it. Those modules were missing from /js_modules and could not be used. Now only the module diff --git a/docs/changes.xml b/docs/changes.xml index 36dea3238..a0488ae8d 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -175,6 +175,22 @@ is moved into its cgroup. + + +an uploaded njs module was written into its file without truncating it, and +without fsync(2). When a longer file of that name was already on disk, for +example one that a failed write had left, the old tail stayed after the new +module. The router read that file and the configuration failed. The upload +was also answered with 200 when the write failed. Now the main process stores +the module through a temporary file, flushes it and renames it, in its store +child, and the upload is answered only when the module is on disk. A deletion +also runs in the store child. A module name can no longer start with ".", and +a module larger than 16 MiB is answered with 413. Earlier versions accepted a +name that starts with "."; such a module is no longer loaded or listed after +the upgrade. + + + at startup, a stored njs module that did not compile hid every stored module diff --git a/src/nxt_controller.c b/src/nxt_controller.c index d42cd5901..4755a63f4 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -39,6 +39,14 @@ typedef struct { #endif +#if (NXT_HAVE_NJS) +typedef struct { + nxt_str_t name; + nxt_controller_request_t *req; +} nxt_controller_script_store_t; +#endif + + typedef struct { nxt_uint_t status; nxt_conf_value_t *conf; @@ -159,6 +167,9 @@ static nxt_bool_t nxt_controller_cert_storing; /* A stored bundle that waits for the first configuration of a router. */ static nxt_controller_cert_store_t *nxt_controller_cert_stored; #endif +#if (NXT_HAVE_NJS) +static nxt_bool_t nxt_controller_script_storing; +#endif static nxt_conf_value_t *nxt_controller_status; @@ -1484,14 +1495,15 @@ nxt_controller_process_request(nxt_task_t *task, nxt_controller_request_t *req) goto invalid_method; } -#if (NXT_TLS) - /* The answer has the certificates. Wait, as in the cert handler. */ - if (nxt_controller_cert_storing) { + /* + * The answer has the certificates and the modules. Wait, as in + * their handlers. + */ + if (nxt_controller_store_in_flight()) { nxt_queue_insert_tail(&nxt_controller_waiting_requests, &req->link); return; } -#endif if (nxt_controller_status == NULL) { nxt_controller_process_status(task, req); @@ -1893,11 +1905,19 @@ nxt_controller_process_config(nxt_task_t *task, nxt_controller_request_t *req, static nxt_bool_t nxt_controller_store_in_flight(void) { + nxt_bool_t storing; + + storing = 0; + #if (NXT_TLS) - return nxt_controller_cert_storing; -#else - return 0; + storing |= nxt_controller_cert_storing; #endif + +#if (NXT_HAVE_NJS) + storing |= nxt_controller_script_storing; +#endif + + return storing; } @@ -2514,15 +2534,16 @@ static void nxt_controller_process_script(nxt_task_t *task, nxt_controller_request_t *req, nxt_str_t *path) { - u_char *p; - nxt_int_t ret; - nxt_str_t name; - nxt_conn_t *c; - nxt_script_t *script; - nxt_buf_mem_t *bm; - nxt_conf_value_t *value; - nxt_controller_response_t resp; - u_char error[NXT_MAX_ERROR_STR]; + u_char *p; + nxt_int_t ret; + nxt_str_t name; + nxt_conn_t *c; + nxt_script_t *script; + nxt_buf_mem_t *bm; + nxt_conf_value_t *value; + nxt_controller_response_t resp; + nxt_controller_script_store_t *store; + u_char error[NXT_MAX_ERROR_STR]; name.length = path->length - 1; name.start = path->start + 1; @@ -2545,6 +2566,13 @@ nxt_controller_process_script(nxt_task_t *task, if (nxt_str_eq(&req->parser.method, "GET", 3)) { + /* While main stores a module, its metadata can roll back. Wait. */ + if (nxt_controller_script_storing) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, + &req->link); + return; + } + if (name.length != 0) { value = nxt_script_info_get(&name); if (value == NULL) { @@ -2572,10 +2600,26 @@ nxt_controller_process_script(nxt_task_t *task, return; } - if (name.length == 0 || path != NULL) { + /* + * The name is a file name in the scripts directory. Main keeps its + * temporary file there under a name that starts with ".", and refuses + * such a name. Answer 400 here, not 500 from main. + */ + if (name.length == 0 || path != NULL || name.start[0] == '.' + || name.length > NXT_SCRIPT_NAME_MAX_LENGTH) + { goto invalid_name; } + /* + * A store or a delete waits for the current reconfiguration: the router + * reads the modules while it applies a configuration. + */ + if (nxt_controller_check_postpone_request(task)) { + nxt_queue_insert_tail(&nxt_controller_waiting_requests, &req->link); + return; + } + if (nxt_str_eq(&req->parser.method, "PUT", 3)) { value = nxt_script_info_get(&name); if (value != NULL) { @@ -2584,12 +2628,23 @@ nxt_controller_process_script(nxt_task_t *task, bm = &c->read->mem; + /* Main refuses a larger module. Send 413 here, not 500. */ + if (nxt_buf_mem_used_size(bm) > NXT_SCRIPT_STORE_MAX_SIZE) { + goto too_large; + } + + store = nxt_mp_get(c->mem_pool, sizeof(nxt_controller_script_store_t)); + if (nxt_slow_path(store == NULL)) { + goto alloc_fail; + } + script = nxt_script_new(task, &name, bm->pos, nxt_buf_mem_used_size(bm), error); if (script == NULL) { goto invalid_script; } + /* A failed store deletes the metadata again. */ ret = nxt_script_info_save(&name, script); nxt_script_destroy(script); @@ -2598,8 +2653,19 @@ nxt_controller_process_script(nxt_task_t *task, goto alloc_fail; } - nxt_script_store_get(task, &name, c->mem_pool, - nxt_controller_process_script_save, req); + nxt_controller_script_storing = 1; + + store->name = name; + store->req = req; + + /* + * Any other change waits until main answers: see + * nxt_controller_store_in_flight(). The request itself is not in + * the waiting queue, so a flush of the queue cannot run it again. + */ + + nxt_script_store_put(task, &name, bm, c->mem_pool, + nxt_controller_process_script_save, store); return; } @@ -2659,6 +2725,15 @@ nxt_controller_process_script(nxt_task_t *task, nxt_controller_response(task, req, &resp); return; +too_large: + + resp.status = 413; + resp.title = (u_char *) "JS module is too large."; + resp.offset = -1; + + nxt_controller_response(task, req, &resp); + return; + script_in_use: resp.status = 400; @@ -2700,38 +2775,36 @@ static void nxt_controller_process_script_save(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { - nxt_conn_t *c; - nxt_buf_mem_t *mbuf; - nxt_controller_request_t *req; - nxt_controller_response_t resp; + nxt_controller_request_t *req; + nxt_controller_response_t resp; + nxt_controller_script_store_t *store; - req = data; + store = data; + req = store->req; + + nxt_controller_script_storing = 0; nxt_memzero(&resp, sizeof(nxt_controller_response_t)); - if (msg == NULL || msg->port_msg.type == _NXT_PORT_MSG_RPC_ERROR) { + if (msg == NULL || msg->port_msg.type != NXT_PORT_MSG_RPC_READY) { + /* + * A store child that failed after rename(2) leaves the file, and + * the next start loads it. + */ + (void) nxt_script_info_delete(&store->name); + resp.status = 500; resp.title = (u_char *) "Failed to store script."; + resp.offset = -1; - nxt_controller_response(task, req, &resp); - return; + } else { + resp.status = 200; + resp.title = (u_char *) "JS module uploaded."; } - c = req->conn; - - mbuf = &c->read->mem; - - nxt_fd_write(msg->fd[0], mbuf->pos, nxt_buf_mem_used_size(mbuf)); - - nxt_fd_close(msg->fd[0]); - msg->fd[0] = -1; - - nxt_memzero(&resp, sizeof(nxt_controller_response_t)); - - resp.status = 200; - resp.title = (u_char *) "JS module uploaded."; - nxt_controller_response(task, req, &resp); + + nxt_controller_flush_requests(task); } diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 0ed03cb06..4d1dd3701 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -959,6 +959,7 @@ static nxt_port_handlers_t nxt_main_process_port_handlers = { #if (NXT_HAVE_NJS) .script_get = nxt_script_store_get_handler, .script_delete = nxt_script_store_delete_handler, + .script_store = nxt_script_store_put_handler, #endif .access_log = nxt_main_port_access_log_handler, .rpc_ready = nxt_port_rpc_handler, @@ -2896,13 +2897,13 @@ nxt_main_store_cancelled(nxt_pid_t pid, int status) * main killed is not a failure: a newer store is pending. A child that * exited before the signal arrived keeps its exit code. * - * A controller that exits while a store child runs starts again here, when - * no store runs or waits. nxt_controller_prefork() reads conf.json and the - * certificate bundles from disk. Started at once, the new controller could - * read them before the child renames its file. It would then keep the old - * state, and main would send the answer of the store to the controller that - * has gone. Only the controller sends stores, so no store is added after - * main has reaped it. + * A controller that exits while a store child runs starts again here, when no + * store runs or waits. nxt_controller_prefork() reads conf.json, the + * certificate bundles and the njs modules from disk. Started at once, the new + * controller could read them before the child renames its file. It would then + * keep the old state, and main would send the answer of the store to the + * controller that has gone. Only the controller sends stores, so no store is + * added after main has reaped it. */ static nxt_bool_t nxt_main_store_exited(nxt_task_t *task, nxt_pid_t pid, int status) diff --git a/src/nxt_port.h b/src/nxt_port.h index 5dc520615..4caf36194 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -80,6 +80,9 @@ struct nxt_port_handlers_s { /* The controller asks main to store a certificate bundle. Appended. */ nxt_port_handler_t cert_store; + + /* The controller asks main to store an njs module. Appended. */ + nxt_port_handler_t script_store; }; @@ -143,6 +146,7 @@ typedef enum { = nxt_port_handler_idx(remove_child_pid), _NXT_PORT_MSG_CERT_STORE = nxt_port_handler_idx(cert_store), + _NXT_PORT_MSG_SCRIPT_STORE = nxt_port_handler_idx(script_store), NXT_PORT_MSG_MAX = sizeof(nxt_port_handlers_t) / sizeof(nxt_port_handler_t), @@ -192,6 +196,7 @@ typedef enum { NXT_PORT_MSG_REMOVE_CHILD_PID = nxt_msg_last(_NXT_PORT_MSG_REMOVE_CHILD_PID), NXT_PORT_MSG_CERT_STORE = nxt_msg_last(_NXT_PORT_MSG_CERT_STORE), + NXT_PORT_MSG_SCRIPT_STORE = nxt_msg_last(_NXT_PORT_MSG_SCRIPT_STORE), } nxt_port_msg_type_t; @@ -201,7 +206,7 @@ typedef enum { * inserted before it fails the build. Add new slots after it and move the * pin to the new last slot. */ -nxt_static_assert(_NXT_PORT_MSG_CERT_STORE == 36, +nxt_static_assert(_NXT_PORT_MSG_SCRIPT_STORE == 37, "a port message slot was inserted, not appended"); diff --git a/src/nxt_script.c b/src/nxt_script.c index fda3ce567..967b82fe1 100644 --- a/src/nxt_script.c +++ b/src/nxt_script.c @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include @@ -32,6 +35,8 @@ static nxt_script_t *nxt_script_get(nxt_task_t *task, nxt_str_t *name, nxt_fd_t fd); static nxt_conf_value_t *nxt_script_details(nxt_mp_t *mp, nxt_script_t *cert); static void nxt_script_buf_completion(nxt_task_t *task, void *obj, void *data); +static nxt_int_t nxt_script_store_name(nxt_buf_t *b, size_t rest, + nxt_str_t *name); static nxt_lvlhsh_t nxt_script_info; @@ -376,7 +381,11 @@ nxt_script_store_load(nxt_task_t *task, nxt_mp_t *mp) name.length = nxt_strlen(de->d_name); name.start = (u_char *) de->d_name; - if (nxt_str_eq(&name, ".", 1) || nxt_str_eq(&name, "..", 2)) { + /* + * No module name starts with ".": this skips "." and "..", and + * the temporary file of a store that did not finish. + */ + if (name.start[0] == '.') { continue; } @@ -528,10 +537,59 @@ nxt_script_buf_completion(nxt_task_t *task, void *obj, void *data) } +/* + * Check the name at the start of "b": it ends with a NUL inside the + * message, and "rest" bytes follow the NUL. A name is not empty, is at most + * NXT_SCRIPT_NAME_MAX_LENGTH bytes, has no "/" and does not start with ".". + * The sender is a process of Unit, but main puts the name into a path. + */ +static nxt_int_t +nxt_script_store_name(nxt_buf_t *b, size_t rest, nxt_str_t *name) +{ + u_char *nul; + nxt_span_t span; + const u_char *start; + + if (b == NULL) { + return NXT_ERROR; + } + + nxt_span_init(&span, b->mem.pos, b->mem.free); + + nul = memchr(b->mem.pos, '\0', nxt_span_len(&span)); + if (nul == NULL) { + return NXT_ERROR; + } + + name->length = nul - b->mem.pos; + + if (nxt_span_take(&span, name->length + 1, &start) != 0 + || nxt_span_len(&span) != rest + || name->length == 0 + || name->length > NXT_SCRIPT_NAME_MAX_LENGTH + || start[0] == '.' + || memchr(start, '/', name->length) != NULL) + { + return NXT_ERROR; + } + + name->start = b->mem.pos; + + return NXT_OK; +} + + +/* + * Only the router reads modules: it opens every module a configuration + * uses. The file is opened read-only and is never created, so a read + * cannot change the store. The controller stores a module with + * nxt_script_store_put(). + */ void nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { u_char *p; + size_t size; nxt_int_t ret; nxt_str_t name; nxt_file_t file; @@ -542,8 +600,8 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) /* * Look up the sender's port via the kernel-validated PID * (SCM_CREDENTIALS). msg->port_msg.pid is self-declared, so using - * it would let a compromised worker spoof the controller / router - * and pull arbitrary script material out of main. + * it would let a compromised worker spoof the router and pull + * arbitrary script material out of main. */ port = nxt_runtime_port_find(task->thread->runtime, nxt_recv_msg_cmsg_pid(msg), @@ -556,10 +614,8 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - if (nxt_slow_path(port->type != NXT_PROCESS_CONTROLLER - && port->type != NXT_PROCESS_ROUTER)) - { - nxt_alert(task, "process %PI cannot store scripts", + if (nxt_slow_path(port->type != NXT_PROCESS_ROUTER)) { + nxt_alert(task, "process %PI cannot read scripts", nxt_recv_msg_cmsg_pid(msg)); nxt_port_recv_msg_close_fds(msg); return; @@ -572,15 +628,24 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) rt = task->thread->runtime; - if (nxt_slow_path(rt->certs.start == NULL)) { + if (nxt_slow_path(rt->scripts.start == NULL)) { nxt_alert(task, "no scripts storage directory"); goto error; } - name.start = msg->buf->mem.pos; - name.length = nxt_strlen(name.start); + if (nxt_slow_path(nxt_script_store_name(msg->buf, 0, &name) != NXT_OK)) { + nxt_alert(task, "script_get_handler: invalid script name"); + goto error; + } + + /* "" and its NUL. */ + if (nxt_slow_path(nxt_size_add(rt->scripts.length, name.length, &size) != 0 + || nxt_size_add(size, 1, &size) != 0)) + { + goto error; + } - file.name = nxt_malloc(rt->scripts.length + name.length + 1); + file.name = nxt_malloc(size); if (nxt_slow_path(file.name == NULL)) { goto error; } @@ -588,7 +653,7 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) p = nxt_cpymem(file.name, rt->scripts.start, rt->scripts.length); p = nxt_cpymem(p, name.start, name.length + 1); - ret = nxt_file_open(task, &file, NXT_FILE_RDWR, NXT_FILE_CREATE_OR_OPEN, + ret = nxt_file_open(task, &file, NXT_FILE_RDONLY, NXT_FILE_OPEN, NXT_FILE_OWNER_ACCESS); nxt_free(file.name); @@ -618,6 +683,205 @@ nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } +/* + * Send "mbuf" to main to store it as the module "name". The module goes in + * a shared memory segment, as a certificate bundle does. The port message + * holds the NUL-terminated name, followed by the size. "handler" gets + * RPC_READY when the file is on disk, and RPC_ERROR or a NULL message + * otherwise. + */ +void +nxt_script_store_put(nxt_task_t *task, nxt_str_t *name, nxt_buf_mem_t *mbuf, + nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx) +{ + void *mem; + size_t size; + uint32_t stream; + nxt_fd_t fd; + nxt_int_t ret; + nxt_buf_t *b; + nxt_port_t *main_port, *ctl_port; + nxt_runtime_t *rt; + + size = nxt_buf_mem_used_size(mbuf); + + fd = nxt_shm_open(task, size); + if (nxt_slow_path(fd == -1)) { + goto fail; + } + + mem = nxt_mem_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + goto fail; + } + + nxt_memcpy(mem, mbuf->pos, size); + nxt_mem_munmap(mem, size); + + b = nxt_buf_mem_alloc(mp, name->length + 1 + sizeof(size_t), 0); + if (nxt_slow_path(b == NULL)) { + goto fail; + } + + b->completion_handler = nxt_script_buf_completion; + + nxt_buf_cpystr(b, name); + *b->mem.free++ = '\0'; + b->mem.free = nxt_cpymem(b->mem.free, &size, sizeof(size_t)); + + rt = task->thread->runtime; + main_port = rt->port_by_type[NXT_PROCESS_MAIN]; + ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; + + stream = nxt_port_rpc_register_handler(task, ctl_port, handler, handler, + -1, ctx); + if (nxt_slow_path(stream == 0)) { + goto fail; + } + + ret = nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_SCRIPT_STORE + | NXT_PORT_MSG_CLOSE_FD, + fd, stream, ctl_port->id, b); + + if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_rpc_cancel(task, ctl_port, stream); + goto fail; + } + + /* See nxt_script_store_get(). */ + nxt_mp_retain(mp); + + return; + +fail: + + /* The port layer owns the descriptor only after a successful write. */ + if (fd != -1) { + nxt_fd_close(fd); + } + + handler(task, NULL, ctx); +} + + +/* + * Main's side of nxt_script_store_put(). The store child of main writes + * the module to "scripts/.store.tmp", flushes it and renames it over + * "scripts/", so a reader sees the old file or the new one, never a + * partial file. Main answers the controller when the child exits. The + * controller is unprivileged, so main checks the sender, the name and the + * size again. + */ +void +nxt_script_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) +{ + size_t size; + u_char *mem; + nxt_str_t name; + nxt_file_t file; + nxt_port_t *port; + nxt_span_t span; + nxt_runtime_t *rt; + nxt_main_store_job_t *job; + + port = nxt_runtime_port_find(task->thread->runtime, + nxt_recv_msg_cmsg_pid(msg), + msg->port_msg.reply_port); + + if (nxt_slow_path(port == NULL || port->type != NXT_PROCESS_CONTROLLER)) { + nxt_alert(task, "process %PI cannot store scripts", + nxt_recv_msg_cmsg_pid(msg)); + nxt_port_recv_msg_close_fds(msg); + return; + } + + rt = task->thread->runtime; + + mem = NULL; + + if (nxt_slow_path(rt->scripts.start == NULL)) { + nxt_alert(task, "no scripts storage directory"); + goto fail; + } + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_alert(task, "script_store_handler: invalid shm fd"); + goto fail; + } + + /* The message holds the NUL-terminated name, followed by the size. */ + + if (nxt_slow_path(nxt_script_store_name(msg->buf, sizeof(size_t), &name) + != NXT_OK)) + { + nxt_alert(task, "script_store_handler: invalid script name"); + goto fail; + } + + nxt_span_init(&span, name.start + name.length + 1, msg->buf->mem.free); + + if (nxt_slow_path(nxt_span_copy(&span, &size, sizeof(size_t)) != 0)) { + nxt_alert(task, "script_store_handler: no script size"); + goto fail; + } + + if (nxt_slow_path(size == 0 || size > NXT_SCRIPT_STORE_MAX_SIZE)) { + nxt_alert(task, "script_store_handler: invalid script size %uz", + size); + goto fail; + } + + /* + * Copy the module with pread() instead of mapping the segment: a + * sender can shrink a mapped segment, and a read past its end is + * SIGBUS in main. A short pread() is only an error. + */ + + mem = nxt_malloc(size); + if (nxt_slow_path(mem == NULL)) { + goto fail; + } + + nxt_memzero(&file, sizeof(nxt_file_t)); + file.fd = msg->fd[0]; + + /* nxt_file_read() logs the name when pread() fails; it must not be NULL. */ + file.name = name.start; + + if (nxt_slow_path(nxt_file_read(&file, mem, size, 0) != (ssize_t) size)) { + nxt_alert(task, "script_store_handler: short script read"); + goto fail; + } + + job = nxt_main_store_job_create(NXT_MAIN_STORE_PUT, "JS module", + &rt->scripts, &name); + if (nxt_slow_path(job == NULL)) { + goto fail; + } + + job->data = mem; + job->size = size; + job->reply_pid = port->pid; + job->reply_port = port->id; + job->stream = msg->port_msg.stream; + + nxt_port_recv_msg_close_fds(msg); + + nxt_main_store_submit(task, job); + + return; + +fail: + + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, -1, + msg->port_msg.stream, 0, NULL); + + nxt_free(mem); + nxt_port_recv_msg_close_fds(msg); +} + + void nxt_script_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) { @@ -640,14 +904,18 @@ nxt_script_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp) } +/* + * The delete is a job of the store child, as a store is. Jobs run in the + * order they come, so a delete cannot run before the rename(2) of an + * earlier store of the same module, which would bring the module back. + */ void nxt_script_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - u_char *p; - nxt_str_t name; - nxt_port_t *ctl_port; - nxt_runtime_t *rt; - nxt_file_name_t *path; + nxt_str_t name; + nxt_port_t *ctl_port; + nxt_runtime_t *rt; + nxt_main_store_job_t *job; rt = task->thread->runtime; ctl_port = rt->port_by_type[NXT_PROCESS_CONTROLLER]; @@ -670,18 +938,16 @@ nxt_script_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) return; } - name.start = msg->buf->mem.pos; - name.length = nxt_strlen(name.start); - - path = nxt_malloc(rt->scripts.length + name.length + 1); - - if (nxt_fast_path(path != NULL)) { - p = nxt_cpymem(path, rt->scripts.start, rt->scripts.length); - p = nxt_cpymem(p, name.start, name.length + 1); + if (nxt_slow_path(nxt_script_store_name(msg->buf, 0, &name) != NXT_OK)) { + nxt_alert(task, "script_delete_handler: invalid script name"); + return; + } - (void) nxt_file_delete(path); + job = nxt_main_store_job_create(NXT_MAIN_STORE_DELETE, "JS module", + &rt->scripts, &name); - nxt_free(path); + if (nxt_fast_path(job != NULL)) { + nxt_main_store_submit(task, job); } } diff --git a/src/nxt_script.h b/src/nxt_script.h index ffefc108f..a300dfb2d 100644 --- a/src/nxt_script.h +++ b/src/nxt_script.h @@ -8,6 +8,16 @@ #define _NXT_SCRIPT_INCLUDED_ +/* + * A module name is a file name in the scripts directory of the state, so + * NAME_MAX bounds it. + */ +#define NXT_SCRIPT_NAME_MAX_LENGTH 255 + +/* Main copies a module into its memory to store it. */ +#define NXT_SCRIPT_STORE_MAX_SIZE (16 * 1024 * 1024) + + typedef struct nxt_script_s nxt_script_t; nxt_script_t *nxt_script_new(nxt_task_t *task, nxt_str_t *name, u_char *data, @@ -25,9 +35,13 @@ void nxt_script_store_release(nxt_array_t *scripts); void nxt_script_store_get(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp, nxt_port_rpc_handler_t handler, void *ctx); +void nxt_script_store_put(nxt_task_t *task, nxt_str_t *name, + nxt_buf_mem_t *mbuf, nxt_mp_t *mp, nxt_port_rpc_handler_t handler, + void *ctx); void nxt_script_store_delete(nxt_task_t *task, nxt_str_t *name, nxt_mp_t *mp); void nxt_script_store_get_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); +void nxt_script_store_put_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); void nxt_script_store_delete_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg); diff --git a/test/test_njs_modules.py b/test/test_njs_modules.py index c322c5f88..a42394e0c 100644 --- a/test/test_njs_modules.py +++ b/test/test_njs_modules.py @@ -1,9 +1,15 @@ +import ctypes import os import shutil +import signal import tempfile +import threading +import time from pathlib import Path -from conftest import unit_run, unit_stop +import pytest + +from conftest import pid_by_name, unit_run, unit_stop from unit.applications.proto import ApplicationProto from unit.option import option @@ -138,6 +144,23 @@ def test_njs_settings_js_module_nul(): } """ +LONG_MODULE = b"""export default { + "route": function() {return 'next'}, + "other": function() {return '%s'} +} +""" % (b'x' * 4096) + + +def route_conf(name): + return { + "settings": {"js_module": name}, + "listeners": {"*:8080": {"pass": "routes/first"}}, + "routes": { + "first": [{"action": {"pass": f"`routes/${{{name}.route()}}`"}}], + "next": [{"action": {"return": 200}}], + }, + } + def start_on(statedir): # main writes here as root; the directory only has to be traversable. @@ -145,6 +168,74 @@ def start_on(statedir): unit_run(state_dir=str(statedir)) +def test_njs_modules_replace_restart(requires_restart): + """A module deleted and uploaded again with a shorter text is stored + and loaded as the short text. Main deletes and stores in its store + child, in the order the requests came.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + stored = statedir / 'scripts' / 'm' + + try: + start_on(statedir) + + assert 'success' in client.conf(LONG_MODULE, '/js_modules/m') + assert stored.read_bytes() == LONG_MODULE, 'long stored' + + assert 'success' in client.conf_delete('/js_modules/m') + assert 'success' in client.conf(SHORT_MODULE, '/js_modules/m') + assert stored.read_bytes() == SHORT_MODULE, 'short stored' + + unit_stop() + start_on(statedir) + + assert 'm' in client.conf_get('/js_modules'), 'loaded on restart' + assert 'success' in client.conf(route_conf('m')) + assert client.get()['status'] == 200 + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +def test_njs_modules_replace_torn_file(requires_restart, skip_alert): + """A module file left torn on disk, as a write cut short leaves it, is + not loaded at startup. Uploading the module again must replace the + whole file, not overwrite its head and keep the old tail.""" + + skip_alert(r'.*JS compile module.*failed.*') + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + scripts = statedir / 'scripts' + stored = scripts / 'm' + + try: + # The first half of LONG_MODULE: a write cut short. + scripts.mkdir() + stored.write_bytes(LONG_MODULE[: len(LONG_MODULE) // 2]) + + start_on(statedir) + + assert 'm' not in client.conf_get('/js_modules'), 'torn not loaded' + + assert 'success' in client.conf(SHORT_MODULE, '/js_modules/m') + + on_disk = stored.read_bytes() + assert on_disk == SHORT_MODULE, f'stored {len(on_disk)} bytes' + + # The router reads the module from the file, not from the upload. + assert 'success' in client.conf(route_conf('m')), 'module compiles' + assert client.get()['status'] == 200 + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + def test_njs_modules_load_after_broken(requires_restart, skip_alert): """One stored module that does not compile must not hide the modules read after it at startup.""" @@ -187,3 +278,253 @@ def test_njs_modules_load_after_broken(requires_restart, skip_alert): finally: unit_stop() shutil.rmtree(statedir, ignore_errors=True) + + +def test_njs_modules_name_dot(): + """Main keeps its temporary file in the scripts directory under a name + that starts with ".", so no module name may start with ".".""" + + for name in ['.x', '.store.tmp']: + resp = client.conf(SHORT_MODULE, f'/js_modules/{name}') + assert resp.get('error') == 'Invalid JS module name.', name + + assert client.conf_get('/js_modules') == {} + + +def test_njs_modules_skip_store_tmp(requires_restart): + """A temporary file that a store left behind is not loaded as a + module.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + scripts = statedir / 'scripts' + + try: + scripts.mkdir() + (scripts / '.store.tmp').write_bytes(SHORT_MODULE) + + start_on(statedir) + + assert client.conf_get('/js_modules') == {} + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +PTRACE_CONT = 7 +PTRACE_DETACH = 17 +PTRACE_GETEVENTMSG = 0x4201 +PTRACE_SEIZE = 0x4206 +PTRACE_INTERRUPT = 0x4207 +PTRACE_O_TRACEFORK = 0x2 +PTRACE_O_TRACEVFORK = 0x4 +PTRACE_O_TRACECLONE = 0x8 +PTRACE_EVENT_STOP = 128 +WALL = 0x40000000 + + +class ForkFreezer(threading.Thread): + """Trace the main process with ptrace(2) and keep the first process it + forks stopped until release(). Every later child runs untouched. + unitd has no test hooks, so this is how a store child is held.""" + + def __init__(self, pid): + super().__init__(daemon=True) + + self.pid = pid + self.libc = ctypes.CDLL(None, use_errno=True) + self.libc.ptrace.restype = ctypes.c_long + self.libc.ptrace.argtypes = [ + ctypes.c_long, + ctypes.c_long, + ctypes.c_void_p, + ctypes.c_void_p, + ] + + self.error = None + self.frozen = None + self.attached = threading.Event() + self.holding = threading.Event() + self.release_req = threading.Event() + self.stop_req = threading.Event() + self.done = threading.Event() + + def ptrace(self, req, pid, addr=0, data=0): + return self.libc.ptrace(req, pid, addr, data) + + def run(self): + try: + self.trace() + finally: + self.done.set() + + def trace(self): + opts = PTRACE_O_TRACEFORK | PTRACE_O_TRACEVFORK | PTRACE_O_TRACECLONE + + if self.ptrace(PTRACE_SEIZE, self.pid, 0, opts) != 0: + self.error = os.strerror(ctypes.get_errno()) + self.attached.set() + return + + self.attached.set() + + released = False + interrupted = False + + while True: + if self.release_req.is_set() and self.holding.is_set(): + if not released: + self.ptrace(PTRACE_DETACH, self.frozen) + released = True + + if self.stop_req.is_set() and not interrupted: + self.ptrace(PTRACE_INTERRUPT, self.pid) + interrupted = True + + pid, status = os.waitpid(self.pid, WALL | os.WNOHANG) + + if pid == 0: + time.sleep(0.002) + continue + + if not os.WIFSTOPPED(status): + return + + sig = os.WSTOPSIG(status) + event = status >> 16 + + if event == 0: + # A signal for main: deliver it. + if interrupted: + self.ptrace(PTRACE_DETACH, self.pid, 0, sig) + return + + self.ptrace(PTRACE_CONT, self.pid, 0, sig) + continue + + if event == PTRACE_EVENT_STOP: + if interrupted: + self.ptrace(PTRACE_DETACH, self.pid) + return + + self.ptrace(PTRACE_CONT, self.pid) + continue + + # A fork, vfork or clone event: a new traced child. + msg = ctypes.c_ulong() + self.ptrace( + PTRACE_GETEVENTMSG, self.pid, 0, ctypes.addressof(msg) + ) + child = msg.value + + # The child reports its first stop; take it before main runs on. + os.waitpid(child, WALL) + + if self.frozen is None: + self.frozen = child + self.holding.set() + + else: + self.ptrace(PTRACE_DETACH, child) + + self.ptrace(PTRACE_CONT, self.pid) + + def release(self): + self.release_req.set() + + def stop(self): + self.release_req.set() + self.stop_req.set() + self.done.wait(10) + + +def test_njs_modules_store_router_restart( + requires_restart, unit_pid, skip_alert, skip_fds_check +): + """The router restarts while main stores a module. The controller must + answer the upload once, when the module is on disk, and must run a + configuration sent meanwhile only after the store.""" + + skip_alert(r'process \d+ exited on signal 9') + skip_fds_check(router=True) + + # The store of the configuration that the start of the test sent must + # not be the child that is held. + time.sleep(0.5) + + controller = pid_by_name('unit: controller') + router = pid_by_name('unit: router') + + freezer = ForkFreezer(unit_pid) + freezer.start() + freezer.attached.wait(5) + + if freezer.error is not None: + freezer.stop() + pytest.skip(f'ptrace: {freezer.error}') + + answers = {} + + def put(key, body, path='/config'): + answers[key] = client.conf(body, path) + answers[f'{key}_time'] = time.monotonic() + + upload = threading.Thread( + target=put, args=('module', SHORT_MODULE, '/js_modules/m') + ) + reconf = threading.Thread( + target=put, + args=( + 'conf', + { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 204}}], + }, + ), + ) + + try: + upload.start() + + assert freezer.holding.wait(10), 'the store child is held' + + os.kill(int(router), signal.SIGKILL) + + for _ in range(100): + new_router = pid_by_name('unit: router') + if new_router not in (None, router): + break + + time.sleep(0.05) + + else: + assert False, 'the router restarted' + + # Let the controller see the new router and send it the config. + time.sleep(1) + + reconf.start() + time.sleep(0.5) + + assert 'module' not in answers, 'no answer before the store' + assert 'conf' not in answers, 'no reconfiguration during the store' + + released = time.monotonic() + freezer.release() + + upload.join(10) + reconf.join(10) + + finally: + freezer.stop() + + assert answers['module'].get('success') == 'JS module uploaded.' + assert answers['module_time'] >= released + assert 'success' in answers['conf'], 'reconfiguration' + assert answers['conf_time'] >= released, 'after the store' + + assert pid_by_name('unit: controller') == controller, 'controller alive' + assert 'm' in client.conf_get('/js_modules') + assert client.get()['status'] == 204 From 60033ee4f1e5abdc3c1d705c587982a6fffc7d2c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 30 Sep 2026 05:22:12 +0200 Subject: [PATCH 364/391] file: flush with F_FULLFSYNC on macOS nxt_file_sync() and nxt_file_dir_sync() were added by commit 9c0ab65c73f29770c82cd98b115e011cd867f865. They call fsync(2). On macOS, fsync(2) sends the data to the drive, but the drive can keep it in its cache, so a power loss can still lose it. The atomic replace of the state files then holds, but the file may not survive a power loss. See https://github.com/freeunitorg/freeunit/issues/516. fcntl(F_FULLFSYNC) also flushes the cache of the drive. It waits for the whole queue of the device, so it is slow: about 22 ms on an M1 drive, and other I/O on the device waits for it. F_BARRIERFSYNC (macOS 10.11 and later) only orders the data before any later write to the device. So nxt_file_sync() now uses F_BARRIERFSYNC, and nxt_file_dir_sync(), which runs after the rename, uses F_FULLFSYNC. That one full flush makes the data and the rename durable. A store flushes the drive cache once for each file, not twice. Without F_BARRIERFSYNC both functions use F_FULLFSYNC. On EINTR the call is repeated. Some file systems do not support these commands: SMB mounts return ENOTSUP, and a file system with no such ioctl returns ENOTTY. On ENOTSUP, EOPNOTSUPP, ENOTTY or EINVAL the functions call fsync(2), as before, and its result counts. Any other error, such as EIO, is returned: the flush failed, so the store keeps the old file. Elsewhere nothing changes: F_FULLFSYNC is defined only on macOS, and there both functions still make one fsync(2) call. ENOTSUP and EOPNOTSUPP are two values on macOS. nxt_file_dir_sync() treated only EOPNOTSUPP and EINVAL as "unsupported", so on an SMB mount it logged an alert for each store. It now uses the same set. Go (internal/poll/fd_fsync_darwin.go) repeats F_FULLFSYNC on EINTR and calls fsync() only on ENOTSUP. SQLite (full_fsync() in src/os_unix.c) calls fsync() after any failed F_FULLFSYNC, which can hide a failed flush. Apple recommends F_BARRIERFSYNC for most writes, and Chromium's base::File::Flush() uses it. No CI leg builds the C code on macOS; unitctl.yml builds only unitctl. The macOS path was not built or run on macOS. On Linux, builds with --hardening=strict and --cc-opt='-DF_FULLFSYNC=51 -DF_BARRIERFSYNC=85' compile it, and Linux answers both unknown fcntl commands with EINVAL. Under strace, ./build/tests made 8 fcntl(fd, 0x55) calls, each on a conf.json.tmp file, and 8 fcntl(fd, 0x33) calls, each on its directory. Each was followed by fsync(2) = 0, and the tests passed. With an LD_PRELOAD shim that makes the calls fail, ./build/tests: - F_BARRIERFSYNC with EIO: "fsync(3, ".../conf.json.tmp") failed (5: Input/output error)", then "main file store test: the first store failed", exit 1. - ENOTSUP (95 on Linux) or ENOTTY: fsync(2) does the flush, exit 0. - F_FULLFSYNC with EIO on the directory: "directory failed (5: Input/output error)" is logged. The store still counts, as before, because the caller ignores the result of nxt_file_dir_sync(). Co-Authored-By: Claude Opus 5.5 --- CHANGES | 6 ++++ docs/changes.xml | 9 +++++ src/nxt_file.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++-- src/nxt_file.h | 3 +- 4 files changed, 107 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index f0505f5f5..54cc67ad2 100644 --- a/CHANGES +++ b/CHANGES @@ -110,6 +110,12 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 its own cgroup as "/scope/python" instead of "/". The namespace is now created after the process is moved into its cgroup. + *) Change: on macOS, a stored state file is now flushed with + fcntl(F_BARRIERFSYNC), and its directory with fcntl(F_FULLFSYNC) + after the rename, so the file also leaves the cache of the drive. + Plain fsync(2) there does not flush that cache, and a power loss + could lose a stored configuration. + *) Security: the router and libunit read records from shared memory that the other process can still write, without enough checks. A buffer that was not a whole number of records was read past its diff --git a/docs/changes.xml b/docs/changes.xml index a0488ae8d..6d0fd16f3 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -199,6 +199,15 @@ not be used. Now only the module that does not compile is left out. + + +on macOS, a stored state file is now flushed with fcntl(F_BARRIERFSYNC), and +its directory with fcntl(F_FULLFSYNC) after the rename, so the file also leaves +the cache of the drive. Plain fsync(2) there does not flush that cache, and a +power loss could lose a stored configuration. + + + when unitd ran as root and the control socket was given to another user or diff --git a/src/nxt_file.c b/src/nxt_file.c index bcc930776..85574de35 100644 --- a/src/nxt_file.c +++ b/src/nxt_file.c @@ -457,12 +457,99 @@ nxt_file_rename(nxt_file_name_t *old_name, nxt_file_name_t *new_name) } +/* + * The errors of a file system that cannot flush a descriptor in the asked + * way. ENOTSUP and EOPNOTSUPP are one value on Linux and FreeBSD, and two + * values on macOS. + */ +static nxt_bool_t +nxt_file_sync_unsupported(nxt_err_t err) +{ + switch (err) { + + case NXT_EINVAL: + case NXT_EOPNOTSUPP: +#if (ENOTSUP != EOPNOTSUPP) + case ENOTSUP: +#endif + case ENOTTY: + return 1; + + default: + return 0; + } +} + + +#if defined(F_FULLFSYNC) + +/* + * Runs a macOS flush command. EINTR repeats the call. If the file system + * does not support the command, for example an SMB mount with ENOTSUP, + * fsync() does the flush and its result counts. Any other error, such as + * EIO, is the result: the flush failed. + */ +static int +nxt_file_fcntl_sync(nxt_fd_t fd, int cmd) +{ + nxt_err_t err; + + for ( ;; ) { + if (fcntl(fd, cmd) == 0) { + return 0; + } + + err = nxt_errno; + + if (err != NXT_EINTR) { + break; + } + } + + if (nxt_file_sync_unsupported(err)) { + return fsync(fd); + } + + /* errno still holds the error of fcntl(). */ + return -1; +} + +#endif + + +/* + * On macOS, fsync(2) sends the data to the drive, but the drive can keep it + * in its cache, so a power loss can still lose it. F_FULLFSYNC also + * flushes the cache of the drive, and it waits for the whole queue of the + * device. F_BARRIERFSYNC only orders the data before any later write to + * the device. So a file gets the barrier, and the directory gets the full + * flush after the rename. That one full flush makes the data and the + * rename durable. Without barrier support fsync() takes the place of the + * barrier, and only the later full flush orders the data, as before. + * Elsewhere both are fsync(). + */ +static int +nxt_file_fsync(nxt_fd_t fd, nxt_bool_t full) +{ +#if defined(F_FULLFSYNC) +#if defined(F_BARRIERFSYNC) + if (!full) { + return nxt_file_fcntl_sync(fd, F_BARRIERFSYNC); + } +#endif + return nxt_file_fcntl_sync(fd, F_FULLFSYNC); +#else + return fsync(fd); +#endif +} + + nxt_int_t nxt_file_sync(nxt_task_t *task, nxt_file_t *file) { nxt_debug(task, "fsync(%FD, \"%FN\")", file->fd, file->name); - if (nxt_fast_path(fsync(file->fd) == 0)) { + if (nxt_fast_path(nxt_file_fsync(file->fd, 0) == 0)) { return NXT_OK; } @@ -492,14 +579,14 @@ nxt_file_dir_sync(nxt_task_t *task, nxt_file_name_t *name) ret = NXT_OK; - if (nxt_slow_path(fsync(fd) != 0)) { + if (nxt_slow_path(nxt_file_fsync(fd, 1) != 0)) { err = nxt_errno; /* * Some file systems refuse to flush a directory descriptor at all; * the rename() has still happened, so this is not a store failure. */ - if (err == NXT_EINVAL || err == NXT_EOPNOTSUPP) { + if (nxt_file_sync_unsupported(err)) { nxt_debug(task, "fsync(%FD, \"%FN\") directory unsupported %E", fd, name, err); diff --git a/src/nxt_file.h b/src/nxt_file.h index 2b63846b6..ed98212e7 100644 --- a/src/nxt_file.h +++ b/src/nxt_file.h @@ -222,7 +222,8 @@ NXT_EXPORT nxt_int_t nxt_file_rename(nxt_file_name_t *old_name, * storage. A durable replace is: write the temporary file, nxt_file_sync() * it, nxt_file_rename() it over the destination, then nxt_file_dir_sync() * the directory that holds them -- without the last step the rename itself - * may not survive a power loss. + * may not survive a power loss. On macOS, nxt_file_sync() only orders the + * data before the rename, and nxt_file_dir_sync() makes both durable. */ NXT_EXPORT nxt_int_t nxt_file_sync(nxt_task_t *task, nxt_file_t *file); NXT_EXPORT nxt_int_t nxt_file_dir_sync(nxt_task_t *task, From 07944c40d702cbdeba988a9b182e935108460217 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 17:11:36 +0200 Subject: [PATCH 365/391] ci: update OpenSSL, move actions off Node 20, pin third-party actions Build OpenSSL 3.6.5 and 4.0.3 in build-test.yml. Both were released on 2026-09-29 and fix CVE-2026-84782 (High), CVE-2026-84783 (Moderate) and several Low issues. See https://openssl-library.org/news/vulnerabilities/ The tarball is not checksum-pinned. The cache key contains the version, so the new version builds fresh. pkg/eol.json and EOL.md declare the tested OpenSSL ceilings. They now say 3.6.5 and 4.0.3 too. Use actions/checkout@v7 in every workflow, and actions/upload-artifact@v7 for the six uses that were on v4. The v4 and v5 forms ran on Node 20, which reached end of life on 2026-04-30. The pinned SHAs in build-deb.yml now point to checkout v7.0.1. No workflow uses pull_request_target or workflow_run, so the new fork-PR block in checkout v7 does not apply. Every upload-artifact name is already unique within its run. Pin the third-party actions that see secrets or publish releases to full commit SHAs: anthropics/claude-code-action v1.0.241, docker/login-action v4.6.0, ncipollo/release-action v1.21.0, taiki-e/install-action v2.87.24 and Swatinem/rust-cache v2.9.2. The version stays in a trailing comment. Raise the floors in test/requirements.txt to pyOpenSSL>=26.0.0 (fixes https://github.com/advisories/GHSA-5pwr-322w-8jr4, CVE-2026-27459) and pytest>=9.0.3 (fixes https://github.com/advisories/GHSA-6w46-j5rx-g56g, CVE-2025-71176). pytest 9 needs Python 3.10 or newer, and every older pytest is affected, so the file has no fallback for older Pythons. The file and test/README.md say so: on an older host, run the tests in a container with test/run-local.sh, or use the distribution's packages. Co-Authored-By: Claude Sonnet 5.5 Reviewed-by: Codex --- .github/workflows/analysis-clang-ast.yml | 2 +- .github/workflows/audit.yml | 10 ++++----- .github/workflows/build-deb.yml | 6 ++--- .github/workflows/build-distros.yml | 6 ++--- .github/workflows/build-test.yml | 28 ++++++++++++------------ .github/workflows/claude.yml | 4 ++-- .github/workflows/eol-check.yml | 2 +- .github/workflows/fuzzing.yml | 4 ++-- .github/workflows/lint-whitespace.yml | 2 +- .github/workflows/release-docker.yml | 12 +++++----- .github/workflows/sanitize.yml | 4 ++-- .github/workflows/security-seccomp.yml | 2 +- .github/workflows/security-syscalls.yml | 4 ++-- .github/workflows/unitctl.yml | 18 +++++++-------- EOL.md | 4 ++-- pkg/eol.json | 4 ++-- test/README.md | 5 +++++ test/requirements.txt | 6 +++-- 18 files changed, 65 insertions(+), 58 deletions(-) diff --git a/.github/workflows/analysis-clang-ast.yml b/.github/workflows/analysis-clang-ast.yml index 5ce5c6a19..12bbfe74b 100644 --- a/.github/workflows/analysis-clang-ast.yml +++ b/.github/workflows/analysis-clang-ast.yml @@ -58,7 +58,7 @@ jobs: [ -f "$f" ] && ln -sf "$f" /usr/lib/$(basename "$f") || true done - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Checkout and build clang-ast run: | diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index db007016a..63ae09a53 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -70,7 +70,7 @@ jobs: outputs: locks: ${{ steps.list.outputs.locks }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: List tracked lock files id: list @@ -89,14 +89,14 @@ jobs: matrix: lock: ${{ fromJSON(needs.locks.outputs.locks) }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: rustup update stable # Prebuilt binary instead of `cargo install cargo-audit`, which would # compile it from source on every run. - name: Install cargo-audit - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 with: tool: cargo-audit @@ -117,7 +117,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: rustup update stable @@ -130,7 +130,7 @@ jobs: python3 -m pytest pkg/contrib/test_check_lock_audit.py - name: Install cargo-audit - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 with: tool: cargo-audit diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index d60a5aa24..06ec578fa 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -85,7 +85,7 @@ jobs: apt-get update apt-get install -y --no-install-recommends git ca-certificates - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -188,7 +188,7 @@ jobs: apt-get update apt-get install -y --no-install-recommends git ca-certificates - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -319,7 +319,7 @@ jobs: apt-get update apt-get install -y --no-install-recommends git ca-certificates - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 8775c5e9b..d8334427d 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -43,7 +43,7 @@ jobs: fi npm install -g node-gyp - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: configure unit CC=${{ matrix.compiler }} run: | @@ -179,7 +179,7 @@ jobs: apk add clang fi - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: configure unit CC=${{ matrix.compiler }} run: | @@ -281,7 +281,7 @@ jobs: exit 1 fi - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: configure unit run: | diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 2c30c4632..b71f37440 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -29,11 +29,11 @@ env: # OpenSSL patch version built from source (see "Build OpenSSL" step). Only the # patch level lives here; the 3.6 "slot" (/opt/openssl-3.6, OPENSSL36_* vars) # is stable across patch bumps and intentionally not templated. - OPENSSL_VERSION: 3.6.2 + OPENSSL_VERSION: 3.6.5 # OpenSSL 4.0 for the openssl4 leg below. No distribution ships 4.0 yet, so # like the 3.6 slot it is built from source; the /opt/openssl-4.0 slot is # stable across patch bumps. - OPENSSL4_VERSION: 4.0.2 + OPENSSL4_VERSION: 4.0.3 concurrency: # Cancel superseded runs for the same pull request. Non-PR events fall back @@ -55,7 +55,7 @@ jobs: outputs: builds: ${{ steps.list.outputs.builds }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Derive build list from pkg/eol.json id: list # Slow legs come first in the list. All 23 legs are created at @@ -94,7 +94,7 @@ jobs: timeout-minutes: 20 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Read njs version id: metadata @@ -155,7 +155,7 @@ jobs: - name: Clone njs repository if: steps.cache-njs.outputs.cache-hit != 'true' - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: repository: freeunitorg/njs ref: '${{ steps.metadata.outputs.njs_version }}' @@ -243,7 +243,7 @@ jobs: run: tar -czf /tmp/unit-core-build.tar.gz Makefile build - name: Upload the core build - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: unit-core-build path: /tmp/unit-core-build.tar.gz @@ -263,7 +263,7 @@ jobs: os: [ubuntu-latest] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 # Provides module, language version and testpath from build name - name: Output build metadata @@ -513,7 +513,7 @@ jobs: - name: Clone njs repository if: steps.cache-njs.outputs.cache-hit != 'true' - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: repository: freeunitorg/njs ref: '${{ steps.metadata.outputs.njs_version }}' @@ -798,7 +798,7 @@ jobs: # Prebuilt binary instead of `cargo install cargo-component` (which # compiles it from source on every run). Same tool, much faster. - name: Install cargo-component - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 with: tool: cargo-component if: steps.metadata.outputs.module == 'wasm-wasi-component' @@ -987,7 +987,7 @@ jobs: timeout-minutes: 20 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Install packages run: | @@ -1039,7 +1039,7 @@ jobs: timeout-minutes: 20 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Install packages run: | @@ -1074,7 +1074,7 @@ jobs: timeout-minutes: 20 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Install packages run: | @@ -1145,7 +1145,7 @@ jobs: timeout-minutes: 30 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Install packages run: | @@ -1229,7 +1229,7 @@ jobs: cc: [gcc, clang] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Install packages run: | diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index afaded0ec..0bde3de3c 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -40,13 +40,13 @@ jobs: actions: read # Required for Claude to read CI results on PRs steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 1 - name: Run Claude Code id: claude - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@cab360f6565aa35a51d6ce9e43f1f4287c0a32ea # v1.0.241 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} diff --git a/.github/workflows/eol-check.yml b/.github/workflows/eol-check.yml index 0484ac4ca..da8338a0e 100644 --- a/.github/workflows/eol-check.yml +++ b/.github/workflows/eol-check.yml @@ -44,7 +44,7 @@ jobs: issues: write # scheduled run may open/update a tracking issue steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 # ubuntu-latest ships a Rust toolchain preinstalled — no setup step needed. # Live-fetch endoflife.date. The CLI degrades gracefully: a single fetch diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index a3b0994af..d90e4741f 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -59,7 +59,7 @@ jobs: timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 120 || 30 }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 # libclang-rt-dev carries libclang_rt.fuzzer, which -fsanitize=fuzzer # links against; clang alone is not enough. @@ -103,7 +103,7 @@ jobs: # A reproducer is the only thing that makes a fuzzing failure actionable. - name: Upload reproducers if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: fuzz-artifacts path: build/fuzz-artifacts/ diff --git a/.github/workflows/lint-whitespace.yml b/.github/workflows/lint-whitespace.yml index c85ae416e..ebd96e834 100644 --- a/.github/workflows/lint-whitespace.yml +++ b/.github/workflows/lint-whitespace.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 33c705cc3..05688f312 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -57,7 +57,7 @@ jobs: variants: ${{ steps.meta.outputs.variants }} steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Resolve version and variant list id: meta @@ -104,7 +104,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Pin version in Dockerfile run: | @@ -115,14 +115,14 @@ jobs: pkg/docker/Dockerfile.${{ matrix.variant }} - name: Log in to GHCR - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ env.GHCR_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Docker Hub - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -160,14 +160,14 @@ jobs: steps: - name: Log in to GHCR - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ env.GHCR_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Docker Hub - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml index 8dacaa57b..956bb13e2 100644 --- a/.github/workflows/sanitize.yml +++ b/.github/workflows/sanitize.yml @@ -199,7 +199,7 @@ jobs: test/test_app_lifecycle.py steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 # Base tools + pytest (needed to run any leg) plus this runtime's extra # apt packages. Runtimes whose toolchain is not in apt add a setup step @@ -378,7 +378,7 @@ jobs: - name: Upload unit logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: unit-logs-${{ matrix.runtime }} path: ${{ env.TMPDIR || '/tmp' }}/unit-test-*/unit.log diff --git a/.github/workflows/security-seccomp.yml b/.github/workflows/security-seccomp.yml index 7db916cc5..e079ab1a9 100644 --- a/.github/workflows/security-seccomp.yml +++ b/.github/workflows/security-seccomp.yml @@ -67,7 +67,7 @@ jobs: timeout-minutes: 20 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - name: Pull test image run: docker pull python:3.13-slim-trixie diff --git a/.github/workflows/security-syscalls.yml b/.github/workflows/security-syscalls.yml index a30823378..c75615615 100644 --- a/.github/workflows/security-syscalls.yml +++ b/.github/workflows/security-syscalls.yml @@ -50,7 +50,7 @@ jobs: libc: [ glibc, musl ] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 # Everything happens inside a pinned container (debian:trixie-slim / # alpine:3.22), so the runner image can move without moving the baseline: @@ -65,7 +65,7 @@ jobs: # and it only works if the list survives a red run. - name: Upload the captured list if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: syscalls-${{ matrix.libc }} path: syscalls-${{ matrix.libc }}.txt diff --git a/.github/workflows/unitctl.yml b/.github/workflows/unitctl.yml index 35099bc24..1c0c7e2c0 100644 --- a/.github/workflows/unitctl.yml +++ b/.github/workflows/unitctl.yml @@ -55,14 +55,14 @@ jobs: os: macos-latest target: aarch64-apple-darwin steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: rustup update stable - run: rustup target add ${{ matrix.target }} - name: Install cross if: matrix.target == 'aarch64-unknown-linux-gnu' - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 with: tool: cross @@ -103,7 +103,7 @@ jobs: run: working-directory: tools/unitctl steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: rustup update stable - run: rustup target add x86_64-unknown-linux-musl @@ -205,18 +205,18 @@ jobs: target: x86_64-apple-darwin steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: rustup update stable - run: rustup target add ${{ matrix.target }} - name: Install cross if: matrix.target == 'aarch64-unknown-linux-gnu' - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e407f7bafb71fd004bc5c2da3032e5470cbb6ef0 # v2.87.24 with: tool: cross - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: prefix-key: rust-${{ matrix.build }} workspaces: ./tools/unitctl -> target @@ -266,13 +266,13 @@ jobs: mv ./target/${{ matrix.target }}/release/unitctl unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }} - name: Upload sha256 sum - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }}.sha256 path: tools/unitctl/unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }}.sha256 - name: Upload unitctl - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }} path: tools/unitctl/unitctl-${{ env.SHORT_VERSION }}-${{ matrix.target }} @@ -289,7 +289,7 @@ jobs: merge-multiple: true - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1.21.0 with: artifacts: "unitctl-*" prerelease: ${{ github.event_name == 'workflow_dispatch' }} diff --git a/EOL.md b/EOL.md index 0b2f159a2..950aa7de8 100644 --- a/EOL.md +++ b/EOL.md @@ -86,8 +86,8 @@ needs an upgrade or replacement decision instead. | Dependency | Role | Floor / Version | Upstream EOL | Notes | |------------|------|-----------------|--------------|-------| | OpenSSL | TLS backend (`--openssl`) | 1.1.1 (floor) | Sep 2023 | The floor stays at 1.1.1 rather than 3.x because RHEL 8 still ships a vendor-patched 1.1.1 and Red Hat maintains it into 2029 -- from 2026-09 it is the last platform in the matrix still receiving vendor patches for it. Amazon Linux 2's ended 2026-06-30 and Debian 11's LTS ends 2026-08-31; both remain inside FreeUnit's own three-year post-EOL support window (see `_grace_os` in `pkg/eol.json`). Floor declared by the `auto/ssltls` probe (PR #224). | -| OpenSSL | tested ceiling | 3.6.2 (CI build) | Nov 2026 | `build-test.yml` builds it into `/opt/openssl-3.6` (`OPENSSL_VERSION`). **The pin needs a bump before 2026-11-01** (3.5 is the LTS, EOL Apr 2030). | -| OpenSSL | tested ceiling | 4.0.2 (CI build) | May 2027 | The `openssl4` job builds it into `/opt/openssl-4.0` (`OPENSSL4_VERSION`) and compiles with `-DOPENSSL_NO_DEPRECATED`, so any use of an API 4.0 deprecates fails the build. | +| OpenSSL | tested ceiling | 3.6.5 (CI build) | Nov 2026 | `build-test.yml` builds it into `/opt/openssl-3.6` (`OPENSSL_VERSION`). **The pin needs a bump before 2026-11-01** (3.5 is the LTS, EOL Apr 2030). | +| OpenSSL | tested ceiling | 4.0.3 (CI build) | May 2027 | The `openssl4` job builds it into `/opt/openssl-4.0` (`OPENSSL4_VERSION`) and compiles with `-DOPENSSL_NO_DEPRECATED`, so any use of an API 4.0 deprecates fails the build. | | Apache Tomcat | Servlet/JSP/EL API + Jasper jars bundled by the Java module | 9.0.x | Mar 2027 ([no earlier than](https://tomcat.apache.org/whichversion.html)) | Bundled by `auto/modules/java`; independent of the JDK variant. | | Eclipse Jetty | `jetty-util` / `jetty-server` / `jetty-http` jars bundled by the Java module | 9.4.58.v20250814 | **Aug 2025 (EOL)** † | Jetty 9.4 lost community support on 2025-08-14 and the bundled build is its last release. Needs an upgrade-or-replace decision — Jetty 10/11 are EOL too; 12.x is the supported line. | | Eclipse ECJ (JDT batch compiler) | JSP compilation jar bundled by the Java module | 3.26.0 | none published | Tracks Eclipse releases; endoflife.date has no product for it. Pinned build is from Jun 2021; current is 3.42.0 (Jun 2025). | diff --git a/pkg/eol.json b/pkg/eol.json index b3aa883d7..a46e144f3 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -96,8 +96,8 @@ "floor_eol": "2023-09", "floor_note": "The floor stays at 1.1.1 rather than 3.x because RHEL 8 (openssl-devel) still ships a vendor-patched 1.1.1 that Red Hat maintains into 2029, and from 2026-09 it is the last platform in the matrix still receiving those patches; Amazon Linux 2 builds against openssl11-devel (see pkg/rpm/Makefile and pkg/rpm/unit.spec.in). Amazon Linux 2 and Debian 11 remain inside FreeUnit's own three-year post-EOL support window (see _grace_os).", "floor_declared_by": "auto/ssltls probe (PR #224), which also drops the pre-1.1.0 compatibility code in src/nxt_openssl.c", - "tested_ceiling": "3.6.2 and 4.0.2", - "tested_ceiling_note": "build-test.yml tests two ceilings from source: 3.6.2 (OPENSSL_VERSION env, built into /opt/openssl-3.6) and 4.0.2 (OPENSSL4_VERSION env, /opt/openssl-4.0, built by the openssl4 job with --cc-opt=-DOPENSSL_NO_DEPRECATED so any use of an API OpenSSL 4.0 deprecates fails the build). ACTION: OpenSSL 3.6 goes upstream-EOL 2026-11-01, so the OPENSSL_VERSION pin needs a bump to a supported branch (3.5 is the LTS, EOL 2030-04); 4.0 is supported to 2027-05.", + "tested_ceiling": "3.6.5 and 4.0.3", + "tested_ceiling_note": "build-test.yml tests two ceilings from source: 3.6.5 (OPENSSL_VERSION env, built into /opt/openssl-3.6) and 4.0.3 (OPENSSL4_VERSION env, /opt/openssl-4.0, built by the openssl4 job with --cc-opt=-DOPENSSL_NO_DEPRECATED so any use of an API OpenSSL 4.0 deprecates fails the build). ACTION: OpenSSL 3.6 goes upstream-EOL 2026-11-01, so the OPENSSL_VERSION pin needs a bump to a supported branch (3.5 is the LTS, EOL 2030-04); 4.0 is supported to 2027-05.", "source": "https://endoflife.date/api/openssl.json" }, "tomcat": { diff --git a/test/README.md b/test/README.md index 7dcb6259a..9d98af2cc 100644 --- a/test/README.md +++ b/test/README.md @@ -106,6 +106,11 @@ core and `fake_upstream` rebuild incrementally. If you prefer to run tests natively (requires all dependencies installed): +The test tools come from the distribution (`python3-pytest`, `python3-openssl`) +or from `pip install -r test/requirements.txt`. That file needs Python 3.10 or +newer, because older pytest and pyOpenSSL releases have known vulnerabilities. +On a host with an older Python, use `./test/run-local.sh` instead. + ```bash # 1. Build FreeUnit with test support ./configure --openssl --njs --zlib --zstd --brotli --otel --tests diff --git a/test/requirements.txt b/test/requirements.txt index 5f94fad23..3d63500cf 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,4 @@ -pyOpenSSL>=20.0.1 -pytest>=6.0.1 +# Needs Python 3.10 or newer (pytest 9). On an older host, run the tests in +# a container with test/run-local.sh instead; see test/README.md. +pyOpenSSL>=26.0.0 +pytest>=9.0.3 From 1453d33cbf1c08cc24854f4c7ba4c1e34b220621 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sat, 3 Oct 2026 12:42:21 +0200 Subject: [PATCH 366/391] http: give each request its own compressor stream The compressor context was one thread-local object for each router thread (nxt_http_comp_compressor_ctx in src/nxt_http_compression.c). But the router compresses an application response in several event loop turns. nxt_router_response_ready_handler() calls nxt_http_comp_compress_app_response() once for each message from the application. Between two of these turns, another response on the same thread can run nxt_http_comp_check_acceptable(), which reset the context, and nxt_http_comp_apply_compression(), which initialised a new stream in it. The last deflate() call of a response freed the stream. So two application responses on one thread used the same stream. A client could get the bytes of another response in a zstd or br body that decodes without an error. Most gzip and deflate bodies were not valid, or stopped early. The router could use a freed stream: it crashed in ZSTD_compressStream2(), and ASan reports a double free of a brotli encoder. Static responses were not affected, because they are compressed in one call. Now nxt_http_comp_check_acceptable() allocates the context from the request pool and stores it in the new r->comp_ctx. The functions that used the thread-local object get the context from the request. nxt_http_comp_wants_compression() and nxt_http_comp_identity_refused() now take the request. The stream is now freed in one place. Each compressor has a new "free" operation, and its deflate() no longer frees the stream. nxt_http_comp_compress() frees the stream after the last call or after a failed call. A request pool cleanup frees it when the response stops early: the client closed the connection, the application failed, or the application timeout expired. Before, such a stream was not freed. A failed zstd call and a static response that failed before its first deflate() call also did not free it. The "live" flag in the context makes sure that a stream is freed only once, and that no call uses it after the free. test/test_python_compression.py streams 8 writes of 4096 bytes from a Python application. Each body holds only a token of its own request. The tests send 24 requests at once with each compressor that the build has, decode each body and look for the tokens of other requests. Two tests also close 4 responses in the middle of the body: one with a client reset, one with the application timeout. On 8 CPUs, with the default router threads, each of the 12 tests failed in 10 of 10 runs before this change, with 21 to 24 of 24 gzip responses bad in each run. With a no-op cleanup, an ASan build reports the 4 streams of each closed test as leaks. The thread-local context came with response compression in 1.35.0. Co-Authored-By: Claude Opus 5.5 Reviewed-by: DeepSeek V4.1 Flash --- CHANGES | 10 + docs/changes.xml | 12 + src/nxt_brotli.c | 18 +- src/nxt_http.h | 3 + src/nxt_http_compression.c | 157 +++++++++--- src/nxt_http_compression.h | 5 +- src/nxt_http_static.c | 4 +- src/nxt_router.h | 1 + src/nxt_zlib.c | 14 +- src/nxt_zstd.c | 10 +- test/python/compression_stream/wsgi.py | 32 +++ test/test_python_compression.py | 339 +++++++++++++++++++++++++ 12 files changed, 547 insertions(+), 58 deletions(-) create mode 100644 test/python/compression_stream/wsgi.py create mode 100644 test/test_python_compression.py diff --git a/CHANGES b/CHANGES index 54cc67ad2..661d449bc 100644 --- a/CHANGES +++ b/CHANGES @@ -57,6 +57,16 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 has the same bytes as before. $body_bytes_sent no longer counts the empty line. + *) Security: with response compression on, the router kept one + compressor stream for each router thread. But it compresses an + application response in several steps. So two application + responses on one thread used the same stream. A client could get + bytes of the response to another client, a body that was not + valid, or a body that stopped early. The router could also crash + when it used a stream that another response had freed. Now each + response has its own stream, and the router frees it when the + response ends or stops early. + *) Change: the main process now writes the state files in a short-lived child process, so a slow fsync(2) no longer delays the start of application processes after a reconfiguration, or diff --git a/docs/changes.xml b/docs/changes.xml index 6d0fd16f3..e878505b9 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -104,6 +104,18 @@ counts the empty line. + + +with response compression on, the router kept one compressor stream for each +router thread. But it compresses an application response in several steps. +So two application responses on one thread used the same stream. A client +could get bytes of the response to another client, a body that was not valid, +or a body that stopped early. The router could also crash when it used a +stream that another response had freed. Now each response has its own stream, +and the router frees it when the response ends or stops early. + + + the main process now writes the state files in a short-lived child diff --git a/src/nxt_brotli.c b/src/nxt_brotli.c index c69d977e0..30c0b840a 100644 --- a/src/nxt_brotli.c +++ b/src/nxt_brotli.c @@ -46,14 +46,14 @@ nxt_brotli_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, &in_len, &in_buf, &out_bytes, &out_buf, NULL); if (!ok) { - goto out_err_free; + return -1; } ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_FLUSH, &in_len, &in_buf, &out_bytes, &out_buf, NULL); if (!ok) { - goto out_err_free; + return -1; } if (last) { @@ -61,18 +61,19 @@ nxt_brotli_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, &in_len, &in_buf, &out_bytes, &out_buf, NULL); if (!ok) { - goto out_err_free; + return -1; } - - BrotliEncoderDestroyInstance(brotli); } return out_len - out_bytes; +} -out_err_free: - BrotliEncoderDestroyInstance(brotli); - return -1; +static void +nxt_brotli_free(nxt_http_comp_compressor_ctx_t *ctx) +{ + BrotliEncoderDestroyInstance(ctx->brotli_ctx); + ctx->brotli_ctx = NULL; } @@ -80,4 +81,5 @@ const nxt_http_comp_operations_t nxt_http_comp_brotli_ops = { .init = nxt_brotli_init, .bound = nxt_brotli_bound, .deflate = nxt_brotli_compress, + .free = nxt_brotli_free, }; diff --git a/src/nxt_http.h b/src/nxt_http.h index 403144e85..351becff0 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -263,6 +263,9 @@ struct nxt_http_request_s { nxt_http_static_ctx_t static_ctx; + /* Set by nxt_http_comp_check_acceptable(), from mem_pool. */ + nxt_http_comp_ctx_t *comp_ctx; + nxt_http_status_t status:16; uint8_t log_route; /* 1 bit */ diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 51fa9e0e9..94daae3e4 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -26,7 +26,6 @@ typedef enum nxt_http_comp_scheme_e nxt_http_comp_scheme_t; typedef struct nxt_http_comp_type_s nxt_http_comp_type_t; typedef struct nxt_http_comp_opts_s nxt_http_comp_opts_t; typedef struct nxt_http_comp_compressor_s nxt_http_comp_compressor_t; -typedef struct nxt_http_comp_ctx_s nxt_http_comp_ctx_t; enum nxt_http_comp_scheme_e { NXT_HTTP_COMP_SCHEME_IDENTITY = 0, @@ -66,6 +65,14 @@ struct nxt_http_comp_compressor_s { nxt_http_comp_opts_t opts; }; +/* + * One per request: nxt_http_comp_check_acceptable() allocates it from the + * request pool and stores it in r->comp_ctx. It was one object per router + * thread. But an application response is compressed in several event loop + * turns, one for each message from the application. So two responses on + * one thread used the same stream. A response could get the bytes of + * another response, or use a stream that the other response had freed. + */ struct nxt_http_comp_ctx_s { nxt_uint_t idx; @@ -96,6 +103,9 @@ struct nxt_http_comp_ctx_s { nxt_off_t resp_clen; nxt_off_t clen_sent; + /* The stream in "ctx" is initialised and not yet freed. */ + bool live; + nxt_http_comp_compressor_ctx_t ctx; }; @@ -113,11 +123,6 @@ struct nxt_http_comp_conf_s { nxt_uint_t nr_enabled; }; -static nxt_thread_declare_data(nxt_http_comp_ctx_t, - nxt_http_comp_compressor_ctx); - -#define nxt_http_comp_ctx() nxt_thread_get_data(nxt_http_comp_compressor_ctx) - static const nxt_conf_map_t nxt_http_comp_compressors_opts_map[] = { { nxt_string("level"), @@ -180,23 +185,62 @@ nxt_http_comp_request_conf(const nxt_http_request_t *r) } +/* + * Frees the stream of one response. This occurs after the last deflate() + * call, after a failed call, and from the request pool cleanup when the + * response stops before its end: the client closed the connection, the + * application failed, or a timer expired. Only the first of these calls + * frees the stream. + */ + +static void +nxt_http_comp_stream_free(nxt_http_comp_ctx_t *ctx) +{ + if (!ctx->live) { + return; + } + + ctx->live = false; + + ctx->type->cops->free(&ctx->ctx); +} + + +static void +nxt_http_comp_ctx_cleanup(nxt_task_t *task, void *obj, void *data) +{ + nxt_http_comp_stream_free(obj); +} + + static ssize_t -nxt_http_comp_compress(uint8_t *dst, size_t dst_size, const uint8_t *src, - size_t src_size, bool last) +nxt_http_comp_compress(nxt_http_comp_ctx_t *ctx, uint8_t *dst, + size_t dst_size, const uint8_t *src, size_t src_size, + bool last) { - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + ssize_t ret; const nxt_http_comp_operations_t *cops; + /* Input after the end of the stream, or after a failed call. */ + if (nxt_slow_path(!ctx->live)) { + return -1; + } + cops = ctx->type->cops; - return cops->deflate(&ctx->ctx, src, src_size, dst, dst_size, last); + ret = cops->deflate(&ctx->ctx, src, src_size, dst, dst_size, last); + + if (last || ret == -1) { + nxt_http_comp_stream_free(ctx); + } + + return ret; } static size_t -nxt_http_comp_bound(size_t size) +nxt_http_comp_bound(nxt_http_comp_ctx_t *ctx, size_t size) { - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); const nxt_http_comp_operations_t *cops; cops = ctx->type->cops; @@ -214,9 +258,9 @@ nxt_http_comp_compress_app_response(nxt_task_t *task, nxt_http_request_t *r, ssize_t cbytes; nxt_buf_t *in, *next, *buf, *out, **tail; nxt_off_t in_len; - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_ctx_t *ctx = r->comp_ctx; - if (ctx->idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + if (ctx == NULL || ctx->idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { return NXT_OK; } @@ -265,14 +309,15 @@ nxt_http_comp_compress_app_response(nxt_task_t *task, nxt_http_request_t *r, * is not part of what bound() promises for the input alone, so the * output buffer gets a small fixed margin on top. */ - buf_len = nxt_http_comp_bound(in_len) + NXT_HTTP_COMP_FLUSH_SLACK; + buf_len = nxt_http_comp_bound(ctx, in_len) + + NXT_HTTP_COMP_FLUSH_SLACK; buf = nxt_buf_mem_ts_alloc(task, in->data, buf_len); if (nxt_slow_path(buf == NULL)) { goto fail; } - cbytes = nxt_http_comp_compress(buf->mem.start, buf_len, + cbytes = nxt_http_comp_compress(ctx, buf->mem.start, buf_len, in->mem.pos, in_len, last); if (nxt_slow_path(cbytes == -1)) { nxt_buf_free(buf->data, buf); @@ -325,12 +370,13 @@ nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, nxt_file_t **f, nxt_file_info_t *fi, size_t static_buf_len, size_t *out_total) { - size_t in_size, out_size, rest; - char *tmp_path, *p; - uint8_t *in, *out; - nxt_int_t ret; - nxt_file_t tfile; - nxt_runtime_t *rt = task->thread->runtime; + size_t in_size, out_size, rest; + char *tmp_path, *p; + uint8_t *in, *out; + nxt_int_t ret; + nxt_file_t tfile; + nxt_runtime_t *rt = task->thread->runtime; + nxt_http_comp_ctx_t *ctx = r->comp_ctx; static const char *template = "unit-compr-XXXXXX"; @@ -357,7 +403,7 @@ nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, tfile.name = (nxt_file_name_t *)tmp_path; in_size = nxt_file_size(fi); - out_size = nxt_http_comp_bound(in_size); + out_size = nxt_http_comp_bound(ctx, in_size); ret = ftruncate(tfile.fd, out_size); if (nxt_slow_path(ret == -1)) { @@ -400,7 +446,8 @@ nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, last = n == rest; - cbytes = nxt_http_comp_compress(out + *out_total, out_size - *out_total, + cbytes = nxt_http_comp_compress(ctx, out + *out_total, + out_size - *out_total, in + in_size - rest, n, last); if (cbytes == -1) { nxt_file_close(task, &tfile); @@ -437,21 +484,22 @@ nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, } +/* + * Tells whether the negotiation chose a compressor for this response. It + * stays true after the stream is freed, so it does not say that a stream is + * live. + */ bool -nxt_http_comp_wants_compression(void) +nxt_http_comp_wants_compression(nxt_http_request_t *r) { - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - - return ctx->idx; + return r->comp_ctx != NULL && r->comp_ctx->idx; } bool -nxt_http_comp_identity_refused(void) +nxt_http_comp_identity_refused(nxt_http_request_t *r) { - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); - - return ctx->identity_refused; + return r->comp_ctx != NULL && r->comp_ctx->identity_refused; } @@ -1252,9 +1300,8 @@ nxt_http_comp_not_acceptable(nxt_http_request_t *r) * this ahead of precondition evaluation: a request that cannot be satisfied * at all must be answered 406, not 304 or 412. The caller therefore asks * this first, evaluates preconditions, and only then applies -- so a 304 - * neither carries a Content-Encoding header nor leaves an initialised - * compressor behind, which would leak, since the compressor is torn down by - * the last deflate() call and a 304 makes none. + * neither carries a Content-Encoding header nor initialises a compressor + * that no deflate() call uses. */ nxt_int_t @@ -1263,10 +1310,14 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) bool identity_refused; nxt_int_t ret, idx; nxt_str_t accept_encoding, mime_type = {}; - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_ctx_t *ctx; nxt_http_comp_conf_t *conf = nxt_http_comp_request_conf(r); - *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1, .sel_idx = -1 }; + /* + * No context means no compression. A context from an earlier call is + * dropped, not reused: its pool cleanup still frees its stream. + */ + r->comp_ctx = NULL; if (r->resp.content_length == NULL && r->resp.content_length_n == -1) { return NXT_OK; @@ -1374,8 +1425,15 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_ERROR; } - ctx->sel_idx = idx; - ctx->identity_refused = identity_refused; + ctx = nxt_mp_get(r->mem_pool, sizeof(nxt_http_comp_ctx_t)); + if (nxt_slow_path(ctx == NULL)) { + return NXT_ERROR; + } + + *ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1, .sel_idx = idx, + .identity_refused = identity_refused }; + + r->comp_ctx = ctx; return NXT_OK; } @@ -1458,10 +1516,14 @@ nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) { int err; nxt_int_t idx; - nxt_http_comp_ctx_t *ctx = nxt_http_comp_ctx(); + nxt_http_comp_ctx_t *ctx = r->comp_ctx; nxt_http_comp_conf_t *conf; nxt_http_comp_compressor_t *compressor; + if (ctx == NULL) { + return NXT_OK; + } + idx = ctx->sel_idx; if (idx == -1 || idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { @@ -1491,11 +1553,26 @@ nxt_http_comp_apply_compression(nxt_task_t *task, nxt_http_request_t *r) ctx->type = compressor->type; ctx->ctx.level = compressor->opts.level; + /* + * Registered before the stream exists, so that no stream is left without + * a cleanup. The cleanup frees the stream when the response stops before + * the last deflate() call. &r->task, not "task": the cleanup keeps the + * pointer, and r->task lives as long as the pool. + */ + if (nxt_slow_path(nxt_mp_cleanup(r->mem_pool, nxt_http_comp_ctx_cleanup, + &r->task, ctx, NULL) + != NXT_OK)) + { + return NXT_ERROR; + } + err = compressor->type->cops->init(&ctx->ctx); if (nxt_slow_path(err)) { return NXT_ERROR; } + ctx->live = true; + return NXT_OK; } diff --git a/src/nxt_http_compression.h b/src/nxt_http_compression.h index c172f2ed3..5cf3a087d 100644 --- a/src/nxt_http_compression.h +++ b/src/nxt_http_compression.h @@ -73,6 +73,7 @@ struct nxt_http_comp_operations_s { ssize_t (*deflate)(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, size_t in_len, uint8_t *out_buf, size_t out_len, bool last); + void (*free)(nxt_http_comp_compressor_ctx_t *ctx); }; @@ -95,8 +96,8 @@ extern nxt_int_t nxt_http_comp_compress_app_response(nxt_task_t *task, extern nxt_int_t nxt_http_comp_compress_static_response(nxt_task_t *task, nxt_http_request_t *r, nxt_file_t **f, nxt_file_info_t *fi, size_t static_buf_len, size_t *out_total); -extern bool nxt_http_comp_wants_compression(void); -extern bool nxt_http_comp_identity_refused(void); +extern bool nxt_http_comp_wants_compression(nxt_http_request_t *r); +extern bool nxt_http_comp_identity_refused(nxt_http_request_t *r); extern bool nxt_http_comp_compressor_is_valid(const nxt_str_t *token); extern nxt_int_t nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r); diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index bc8ec11e1..b4e0621a0 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -976,7 +976,7 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, * representation it refused. */ - if (nxt_http_comp_identity_refused()) { + if (nxt_http_comp_identity_refused(r)) { rstatus = NXT_HTTP_OK; } @@ -1105,7 +1105,7 @@ nxt_http_static_send(nxt_task_t *task, nxt_http_request_t *r, goto fail; } - if (nxt_http_comp_wants_compression()) { + if (nxt_http_comp_wants_compression(r)) { size_t out_total; nxt_int_t ret; diff --git a/src/nxt_router.h b/src/nxt_router.h index fbe7b27f9..735c21c52 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -25,6 +25,7 @@ typedef struct nxt_upstreams_s nxt_upstreams_t; typedef struct nxt_router_access_log_s nxt_router_access_log_t; typedef struct nxt_router_access_log_format_s nxt_router_access_log_format_t; typedef struct nxt_http_comp_conf_s nxt_http_comp_conf_t; +typedef struct nxt_http_comp_ctx_s nxt_http_comp_ctx_t; #define NXT_HTTP_ACTION_ERROR ((nxt_http_action_t *) -1) diff --git a/src/nxt_zlib.c b/src/nxt_zlib.c index ad1957ead..ea2871b88 100644 --- a/src/nxt_zlib.c +++ b/src/nxt_zlib.c @@ -62,22 +62,25 @@ nxt_zlib_deflate(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, ret = deflate(z, last ? Z_FINISH : Z_SYNC_FLUSH); if (ret == Z_STREAM_ERROR || ret == Z_BUF_ERROR) { - deflateEnd(z); return -1; } - if (last) { - deflateEnd(z); - } - return z->total_out - compressed_bytes; } +static void +nxt_zlib_free(nxt_http_comp_compressor_ctx_t *ctx) +{ + deflateEnd(&ctx->zlib_ctx); +} + + const nxt_http_comp_operations_t nxt_http_comp_deflate_ops = { .init = nxt_zlib_deflate_init, .bound = nxt_zlib_bound, .deflate = nxt_zlib_deflate, + .free = nxt_zlib_free, }; @@ -85,4 +88,5 @@ const nxt_http_comp_operations_t nxt_http_comp_gzip_ops = { .init = nxt_zlib_gzip_init, .bound = nxt_zlib_bound, .deflate = nxt_zlib_deflate, + .free = nxt_zlib_free, }; diff --git a/src/nxt_zstd.c b/src/nxt_zstd.c index 8fb379448..b4d8ad2cc 100644 --- a/src/nxt_zstd.c +++ b/src/nxt_zstd.c @@ -51,7 +51,6 @@ nxt_zstd_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, if (last) { ret = ZSTD_endStream(zstd, &zoutb); - ZSTD_freeCStream(zstd); } if (ZSTD_isError(ret)) { @@ -62,8 +61,17 @@ nxt_zstd_compress(nxt_http_comp_compressor_ctx_t *ctx, const uint8_t *in_buf, } +static void +nxt_zstd_free(nxt_http_comp_compressor_ctx_t *ctx) +{ + ZSTD_freeCStream(ctx->zstd_ctx); + ctx->zstd_ctx = NULL; +} + + const nxt_http_comp_operations_t nxt_http_comp_zstd_ops = { .init = nxt_zstd_init, .bound = nxt_zstd_bound, .deflate = nxt_zstd_compress, + .free = nxt_zstd_free, }; diff --git a/test/python/compression_stream/wsgi.py b/test/python/compression_stream/wsgi.py new file mode 100644 index 000000000..a885e1daa --- /dev/null +++ b/test/python/compression_stream/wsgi.py @@ -0,0 +1,32 @@ +import time + + +def application(environ, start_response): + # The body holds only the token of this request, so the bytes of another + # response are easy to see. It is sent in several writes with a pause + # between them. Each write is one message to the router, so the router + # compresses the body in several event loop turns. + rid = int(environ.get('HTTP_X_ID', '0')) + parts = int(environ.get('HTTP_X_PARTS', '8')) + size = int(environ.get('HTTP_X_SIZE', '4096')) + delay = float(environ.get('HTTP_X_DELAY', '0.02')) + + token = b'<%06d>' % rid + part = (token * (size // len(token) + 1))[:size] + + start_response( + '200 OK', + [ + ('Content-Type', 'text/plain'), + ('Content-Length', str(size * parts)), + ], + ) + + def body(): + for i in range(parts): + if i: + time.sleep(delay) + + yield part + + return body() diff --git a/test/test_python_compression.py b/test/test_python_compression.py new file mode 100644 index 000000000..2542d8a86 --- /dev/null +++ b/test/test_python_compression.py @@ -0,0 +1,339 @@ +import gzip +import re +import socket +import struct +import threading +import time +import zlib + +import pytest + +from unit.applications.lang.python import ApplicationPython + +prerequisites = {'modules': {'python': 'any'}} + +client = ApplicationPython() + +ENCODINGS = ['gzip', 'deflate', 'zstd', 'br'] + +# Concurrent responses per test. The bug needs two responses on one router +# thread, and the router has one thread per CPU. On 8 CPUs, 24 responses +# made the test fail in each of 10 runs before the fix. +CONCURRENCY = 24 +PARTS = 8 +SIZE = 4096 + +TOKEN = re.compile(rb'<(\d{6})>') + + +def decoder(encoding): + """Returns a function that decodes a whole body, or skips the test.""" + + if encoding == 'gzip': + return gzip.decompress + + if encoding == 'deflate': + return zlib.decompress + + if encoding == 'zstd': + try: + from compression import zstd # Python 3.14 and later. + + return zstd.decompress + except ImportError: + pass + + zstandard = pytest.importorskip('zstandard') + + def decode(data): + d = zstandard.ZstdDecompressor().decompressobj() + out = d.decompress(data) + + if not d.eof: + raise ValueError('zstd frame is not complete') + + return out + + return decode + + return pytest.importorskip('brotli').decompress + + +def setup_compression(encoding, **kwargs): + client.load('compression_stream', threads=32, **kwargs) + + resp = client.conf( + { + "http": { + "compression": { + "types": ["text/plain"], + "compressors": [{"encoding": encoding}], + } + } + }, + 'settings', + ) + + if 'success' in resp: + return + + if 'supported compressor' in resp.get('detail', ''): + pytest.skip(f'unit built without {encoding} compression support') + + pytest.fail(f'could not configure compression: {resp}') + + +# The app sleeps "delay" seconds between writes, so the responses on one +# router thread take turns. The stream tests depend on that: a machine that +# finishes a response in one turn can pass them without the fix. Before the +# fix, 10 of 10 runs failed, with 21 to 24 of 24 bodies bad. +def request_headers(rid, encoding, parts, delay=0.02): + return { + 'Host': 'localhost', + 'Accept-Encoding': encoding, + 'X-Id': str(rid), + 'X-Parts': str(parts), + 'X-Size': str(SIZE), + 'X-Delay': str(delay), + 'Connection': 'close', + } + + +def expected(rid, parts=PARTS): + token = b'<%06d>' % rid + + return (token * (SIZE // len(token) + 1))[:SIZE] * parts + + +def dechunk(data): + body = b'' + + while True: + end = data.find(b'\r\n') + if end == -1: + raise ValueError('the response ends before its last chunk') + + size = int(data[:end], 16) + if size == 0: + return body + + chunk = data[end + 2 : end + 2 + size] + if len(chunk) != size: + raise ValueError('the response ends inside a chunk') + + body += chunk + data = data[end + 2 + size + 2 :] + + +def fetch(rid, encoding): + # latin1 maps each byte to one character, so encoding back gives the + # compressed bytes exactly. + resp = client.get( + headers=request_headers(rid, encoding, PARTS), + raw_resp=True, + encoding='latin1', + read_buffer_size=65536, + ) + + head, _, body = resp.partition('\r\n\r\n') + lines = head.split('\r\n') + + headers = {} + for line in lines[1:]: + name, _, value = line.partition(':') + headers[name.strip()] = value.strip() + + return int(lines[0].split()[1]), headers, body.encode('latin1') + + +def check(rid, encoding, decode): + """Returns None for a correct response, else what is wrong with it.""" + + status, headers, body = fetch(rid, encoding) + + if status != 200: + return f'status {status}' + + if headers.get('Content-Encoding') != encoding: + return f'Content-Encoding {headers.get("Content-Encoding")}' + + # The length of the coded body is not known in advance, so the router + # sends it chunked. + try: + if headers.get('Transfer-Encoding') == 'chunked': + body = dechunk(body) + + data = decode(body) + except Exception as e: # pylint: disable=broad-except + return f'not valid {encoding}: {e!r}' + + foreign = {int(m) for m in TOKEN.findall(data)} - {rid} + if foreign: + return f'holds bytes of requests {sorted(foreign)[:4]}' + + if data != expected(rid): + return f'{len(data)} bytes, expected {len(expected(rid))}' + + return None + + +def burst(encoding, decode, first_rid): + """Sends CONCURRENCY requests at once; returns the bad responses.""" + + results = {} + barrier = threading.Barrier(CONCURRENCY) + + def worker(rid): + try: + barrier.wait(timeout=30) + results[rid] = check(rid, encoding, decode) + except BaseException as e: # pylint: disable=broad-except + results[rid] = repr(e) + + threads = [ + threading.Thread(target=worker, args=(rid,)) + for rid in range(first_rid, first_rid + CONCURRENCY) + ] + + for t in threads: + t.start() + + for t in threads: + t.join(timeout=120) + + assert len(results) == CONCURRENCY, 'every request finished' + + return {rid: why for rid, why in sorted(results.items()) if why} + + +def describe(bad): + return f'{len(bad)} of {CONCURRENCY} bad, first: {list(bad.items())[:3]}' + + +def start_streams(encoding, parts, delay): + """Starts 4 compressed responses and waits for the header of each.""" + + socks = [] + + for rid in range(1001, 1005): + sock = client.get( + headers=request_headers(rid, encoding, parts, delay), + no_recv=True, + ) + sock.settimeout(30) + socks.append(sock) + + # The router initialises the stream before it sends the header. The + # empty line at the end of the header goes out with the first chunk, + # which for zstd can be the last one, so do not wait for it. + coded = b'\r\nContent-Encoding: ' + encoding.encode() + b'\r\n' + + for sock in socks: + data = b'' + while coded not in data: + part = sock.recv(4096) + assert part, 'the response starts' + data += part + + return socks + + +def burst_with(encoding, decode, action): + """Runs action() while a burst of responses streams.""" + + result = {} + + def run_burst(): + result['bad'] = burst(encoding, decode, 1) + + t = threading.Thread(target=run_burst) + t.start() + + # Let the responses of the burst start to stream. + time.sleep(0.05) + + action() + + t.join(timeout=120) + + assert 'bad' in result, 'the burst finished' + + return result['bad'] + + +@pytest.mark.parametrize('encoding', ENCODINGS) +def test_python_compression_concurrent_streams(encoding): + # The compressor state was one object per router thread. A response that + # is compressed in several event loop turns shared it with each other + # response on that thread. A client then got invalid data, the bytes of + # another response, or a router crash. + decode = decoder(encoding) + setup_compression(encoding) + + bad = burst(encoding, decode, 1) + + assert not bad, describe(bad) + + +@pytest.mark.parametrize('encoding', ENCODINGS) +def test_python_compression_client_close(encoding): + # Clients that close the connection in the middle of a compressed body, + # while other compressed responses stream. The router must free the + # stream of each closed response (an ASan build reports a leak if it does + # not), and must not free or use the stream of another response. + decode = decoder(encoding) + setup_compression(encoding) + + socks = start_streams(encoding, 40, 0.02) + + def reset(): + for sock in socks: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0) + ) + sock.close() + + bad = burst_with(encoding, decode, reset) + + assert not bad, describe(bad) + + # The application ends the closed responses after 40 parts. Wait for + # that, then check that the router still serves a correct response. + time.sleep(1) + + assert check(2001, encoding, decode) is None, 'response after the close' + + +@pytest.mark.parametrize('encoding', ENCODINGS) +def test_python_compression_app_timeout(encoding): + # The application stops for longer than "limits": {"timeout"} in the + # middle of a compressed body. The router then closes the connection, + # and must free the stream of the response. A client close does not do + # this for zstd: the router writes no zstd data until the last part, so + # it does not see the close before the stream ends. + decode = decoder(encoding) + setup_compression(encoding, limits={"timeout": 1}) + + socks = start_streams(encoding, 2, 2) + + bad = burst_with(encoding, decode, lambda: None) + + assert not bad, describe(bad) + + for sock in socks: + data = b'' + while True: + part = sock.recv(65536) + if not part: + break + data += part + + sock.close() + + assert not data.endswith(b'0\r\n\r\n'), 'the response is cut' + + # The application ends the slow responses 2 seconds after it starts + # them. Wait for that, then check a response again. + time.sleep(1) + + assert check(2001, encoding, decode) is None, 'response after the timeout' From 504ec33cb42ab96ee869661083ad3462dda73894 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 12:35:58 +0200 Subject: [PATCH 367/391] http: do not compress when response_headers set Content-Encoding The router decides on compression before "response_headers" are applied. nxt_http_comp_check_acceptable() returned early only for a Content-Encoding that was already in the response fields. A Content-Encoding from "response_headers" is added later, in nxt_http_request_header_send(), and nxt_http_set_headers() then replaced the value of the field that the compressor had added. So a share of "$uri.gz" with "Content-Encoding: gzip" in "response_headers" sent gzip of the stored gzip file to a client that accepts gzip. The field named one coding, and the ETag was weak. An application response that sets its coding this way had the same fault. A Content-Encoding removed with null in "response_headers" sent gzip bytes with no Content-Encoding. The new nxt_http_set_headers_field_op() says what "response_headers" will do to a field. It matches the name without case, uses the same status range as nxt_http_set_headers(), and lets the last matching key win. The check uses it for both callers, the static handler and the router. A template value is resolved at the check, with the code that nxt_http_set_headers() uses. The new nxt_http_set_headers_value() holds it for both. It resolves each key once per request and stores the result on the request, so nxt_http_set_headers() does not resolve that key again. A JavaScript template therefore runs once, and the check and the header see the same value. A value that is not safe in a field is not sent, so it counts as absent, and an earlier key can then give the result. A null value removes the field. A variable that is not set gives an empty string. An empty string is a value, and the field is sent empty, as before. A failed query makes the check return NXT_ERROR. A cacheable variable in such a template keeps the value from the check for the rest of the request, including the access log. A configured value is now treated like a Content-Encoding in the response: nothing is compressed, the ETag stays strong, and Vary is not merged. A configured removal keeps the compressor out. The body is then identity, so a client that refused identity gets 406. This applies only to a coding that Unit would apply. When the response has its own Content-Encoding, a configured removal still removes only the field. Closes https://github.com/freeunitorg/freeunit/issues/557 Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 Reviewed-by: Codex Reviewed-by: DeepSeek --- CHANGES | 14 ++ docs/changes.xml | 17 ++ src/nxt_http.h | 15 ++ src/nxt_http_compression.c | 48 ++++- src/nxt_http_set_headers.c | 298 +++++++++++++++++++++++---- test/python/content_encoding/wsgi.py | 15 ++ test/test_python_compression.py | 136 ++++++++++++ test/test_static_compression.py | 211 ++++++++++++++++++- 8 files changed, 703 insertions(+), 51 deletions(-) create mode 100644 test/python/content_encoding/wsgi.py diff --git a/CHANGES b/CHANGES index 661d449bc..5df246419 100644 --- a/CHANGES +++ b/CHANGES @@ -940,6 +940,20 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 the acceptability check other than success as a server error; only the static path reported the 406. + *) Bugfix: a "Content-Encoding" set with "response_headers" no longer + makes the body coded twice. With a compressor configured, the + response was compressed first, and "response_headers" then replaced + the "Content-Encoding" of the compressor. So a share of "$uri.gz" + with "Content-Encoding: gzip" sent gzip of the stored gzip file to a + client that accepts gzip, with a weak ETag. Now such a response is + not compressed, as with a "Content-Encoding" from an application. + This applies to static files and to application responses. A + "Content-Encoding" removed with null in "response_headers" also keeps + the compressor out, so bytes that Unit compressed are no longer sent + without a "Content-Encoding". Such a response is identity, so a + client that refused identity gets "406 Not Acceptable". For a + response that has its own "Content-Encoding", the null still removes + only the field, as before. *) Change: unitd, libunit.a and the language modules no longer contain the test hooks compiled under "#if (NXT_TESTS)". "./configure diff --git a/docs/changes.xml b/docs/changes.xml index e878505b9..359742979 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -1261,6 +1261,23 @@ static path reported the 406. + + +a "Content-Encoding" set with "response_headers" no longer makes the body +coded twice. With a compressor configured, the response was compressed first, +and "response_headers" then replaced the "Content-Encoding" of the compressor. +So a share of "$uri.gz" with "Content-Encoding: gzip" sent gzip of the stored +gzip file to a client that accepts gzip, with a weak ETag. Now such a +response is not compressed, as with a "Content-Encoding" from an application. +This applies to static files and to application responses. A +"Content-Encoding" removed with null in "response_headers" also keeps the +compressor out, so bytes that Unit compressed are no longer sent without a +"Content-Encoding". Such a response is identity, so a client that refused +identity gets "406 Not Acceptable". For a response that has its own +"Content-Encoding", the null still removes only the field, as before. + + + unitd, libunit.a and the language modules no longer contain the test hooks diff --git a/src/nxt_http.h b/src/nxt_http.h index 351becff0..49531ae70 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -65,6 +65,18 @@ typedef enum { } nxt_http_te_t; +/* What "response_headers" will do to one response field. */ +typedef enum { + NXT_HTTP_SET_HEADER_NONE = 0, + NXT_HTTP_SET_HEADER_REPLACE, + NXT_HTTP_SET_HEADER_REMOVE, + NXT_HTTP_SET_HEADER_ERROR, +} nxt_http_set_header_op_t; + + +typedef struct nxt_http_set_headers_ctx_s nxt_http_set_headers_ctx_t; + + typedef enum { NXT_HTTP_PROTO_H1 = 0, NXT_HTTP_PROTO_H2, @@ -240,6 +252,7 @@ struct nxt_http_request_s { nxt_tstr_query_t *tstr_query; nxt_tstr_cache_t tstr_cache; + nxt_http_set_headers_ctx_t *set_headers; nxt_http_action_t *action; void *req_rpc_data; @@ -509,6 +522,8 @@ nxt_int_t nxt_http_rewrite_init(nxt_router_conf_t *rtcf, nxt_int_t nxt_http_rewrite(nxt_task_t *task, nxt_http_request_t *r); nxt_bool_t nxt_http_set_headers_override_validators(nxt_http_request_t *r); +nxt_http_set_header_op_t nxt_http_set_headers_field_op(nxt_http_request_t *r, + const char *name, size_t length); nxt_int_t nxt_http_comp_merge_vary(nxt_http_request_t *r); nxt_int_t nxt_http_set_headers_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf); diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c index 94daae3e4..8d800f5f5 100644 --- a/src/nxt_http_compression.c +++ b/src/nxt_http_compression.c @@ -1307,11 +1307,12 @@ nxt_http_comp_not_acceptable(nxt_http_request_t *r) nxt_int_t nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) { - bool identity_refused; - nxt_int_t ret, idx; - nxt_str_t accept_encoding, mime_type = {}; - nxt_http_comp_ctx_t *ctx; - nxt_http_comp_conf_t *conf = nxt_http_comp_request_conf(r); + bool identity_refused; + nxt_int_t ret, idx; + nxt_str_t accept_encoding, mime_type = {}; + nxt_http_comp_ctx_t *ctx; + nxt_http_comp_conf_t *conf = nxt_http_comp_request_conf(r); + nxt_http_set_header_op_t conf_op; /* * No context means no compression. A context from an earlier call is @@ -1350,6 +1351,39 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return NXT_OK; } + /* + * "response_headers" are applied after this, when the header is sent. + * A Content-Encoding there then replaces the one that a compressor adds, + * or removes it. Both make a coded body wrong: coded twice under one + * coding name, or coded under no name at all (#557). + * + * A value means that the operator says the body is already coded. This + * is the usual way to serve a precompressed file. It is the same case + * as a Content-Encoding in the response, so it returns here too. + * + * A null value removes the field. The body then has no coding, so it is + * identity, and a refusal of identity is still answered below. Only the + * compressor is kept out. This applies only to a coding that Unit would + * apply. A response that has its own Content-Encoding returned above, + * and a null value then only removes that field from the coded body. + * + * A template value is resolved now, once for the request. A variable + * that is not set gives an empty string, and an empty string is a value. + * A value that is not safe in a field is not sent, so it counts as + * absent. + */ + + conf_op = nxt_http_set_headers_field_op(r, "Content-Encoding", + nxt_length("Content-Encoding")); + + if (nxt_slow_path(conf_op == NXT_HTTP_SET_HEADER_ERROR)) { + return NXT_ERROR; + } + + if (conf_op == NXT_HTTP_SET_HEADER_REPLACE) { + return NXT_OK; + } + ret = nxt_http_comp_accept_encoding(r, &accept_encoding); if (nxt_slow_path(ret != NXT_OK)) { return NXT_ERROR; @@ -1370,6 +1404,10 @@ nxt_http_comp_check_acceptable(nxt_task_t *task, nxt_http_request_t *r) return nxt_http_comp_not_acceptable(r); } + if (conf_op == NXT_HTTP_SET_HEADER_REMOVE) { + return identity_refused ? nxt_http_comp_not_acceptable(r) : NXT_OK; + } + /* * A built configuration always holds identity. So NULL is the only "no * compression" state, and identity is then the only coding. A client diff --git a/src/nxt_http_set_headers.c b/src/nxt_http_set_headers.c index 2ac6c696e..9caef0d1f 100644 --- a/src/nxt_http_set_headers.c +++ b/src/nxt_http_set_headers.c @@ -14,6 +14,28 @@ typedef struct { } nxt_http_header_val_t; +/* The keys of r->action resolved for this request, one entry per key. */ + +struct nxt_http_set_headers_ctx_s { + nxt_http_action_t *action; + nxt_str_t *value; + uint8_t *state; +}; + + +typedef enum { + NXT_HTTP_SET_HEADERS_UNRESOLVED = 0, + NXT_HTTP_SET_HEADERS_RESOLVED, + NXT_HTTP_SET_HEADERS_REJECTED, +} nxt_http_set_headers_state_t; + + +static nxt_http_set_headers_ctx_t *nxt_http_set_headers_ctx( + nxt_http_request_t *r); +static nxt_int_t nxt_http_set_headers_value(nxt_http_request_t *r, + nxt_http_set_headers_ctx_t *ctx, nxt_uint_t i); + + /* * Whether the matched action replaces or removes one of the validators the * static handler generates. @@ -26,9 +48,7 @@ typedef struct { * evaluate preconditions at all in that case, which loses the 304 but is never * wrong. * - * Only the name matters here. The value may be a template resolved per - * request, and resolving it this early would move tstr queries ahead of where - * nxt_http_set_headers() runs them. + * Only the name matters here, so no template value is resolved. */ nxt_bool_t @@ -68,6 +88,93 @@ nxt_http_set_headers_override_validators(nxt_http_request_t *r) } +/* + * What nxt_http_set_headers() will later do to the response field "name". + * + * Code that runs before the header is sent can ask this. Compression needs + * it: a Content-Encoding from "response_headers" replaces the one that a + * compressor adds. So the compressor must not run. Otherwise the body is + * coded twice and the field names one coding only. + * + * The value is resolved here with nxt_http_set_headers_value(). It stores + * the result on the request, and nxt_http_set_headers() uses that result. + * So each key is resolved once, and both see the same value, also for a + * JavaScript template. A template value can resolve to bytes that are not + * safe in a field. nxt_http_set_headers() then skips that key, so it + * counts as absent here too. A null value removes the field. An empty + * string is a value: the field is sent empty. A variable that is not set + * gives an empty string. A failed query gives NXT_HTTP_SET_HEADER_ERROR. + * + * Resolving a template this early is safe. The query is synchronous. + * nxt_tstr_query_init() reuses r->tstr_query. A cacheable variable keeps + * its value in r->tstr_cache for the rest of the request. + * + * The status test is the same as in nxt_http_set_headers(). Keys are + * applied in order, so the last key that matches and is not skipped gives + * the result. The search goes back from the end and stops at that key. + */ + +nxt_http_set_header_op_t +nxt_http_set_headers_field_op(nxt_http_request_t *r, const char *name, + size_t length) +{ + nxt_int_t ret; + nxt_uint_t i; + nxt_http_action_t *action; + nxt_http_header_val_t *header; + nxt_http_set_headers_ctx_t *ctx; + + action = r->action; + + if (action == NULL || action->set_headers == NULL) { + return NXT_HTTP_SET_HEADER_NONE; + } + + if (r->status < NXT_HTTP_OK || r->status >= NXT_HTTP_BAD_REQUEST) { + return NXT_HTTP_SET_HEADER_NONE; + } + + /* The context is made only when a key matches. */ + + ctx = NULL; + + header = action->set_headers->elts; + i = action->set_headers->nelts; + + while (i > 0) { + i--; + + if (header[i].name.length != length + || nxt_memcasecmp(header[i].name.start, name, length) != 0) + { + continue; + } + + if (ctx == NULL) { + ctx = nxt_http_set_headers_ctx(r); + if (nxt_slow_path(ctx == NULL)) { + return NXT_HTTP_SET_HEADER_ERROR; + } + } + + ret = nxt_http_set_headers_value(r, ctx, i); + + if (ret == NXT_DECLINED) { + continue; + } + + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_HTTP_SET_HEADER_ERROR; + } + + return (ctx->value[i].start != NULL) ? NXT_HTTP_SET_HEADER_REPLACE + : NXT_HTTP_SET_HEADER_REMOVE; + } + + return NXT_HTTP_SET_HEADER_NONE; +} + + nxt_int_t nxt_http_set_headers_init(nxt_router_conf_t *rtcf, nxt_http_action_t *action, nxt_http_action_conf_t *acf) @@ -153,6 +260,126 @@ nxt_http_header_value_is_safe(const nxt_str_t *v) } +/* + * The resolved keys of r->action for this request. The context is made on + * first use and kept in r->set_headers. A request is allocated zeroed for + * each request, also on a keep-alive connection, so the pointer never comes + * from an earlier request. The context is made again if r->action is not + * the action that it was made for. + */ + +static nxt_http_set_headers_ctx_t * +nxt_http_set_headers_ctx(nxt_http_request_t *r) +{ + nxt_uint_t n; + nxt_http_set_headers_ctx_t *ctx; + + ctx = r->set_headers; + + /* + * Today the action does not change between the compression check and + * the header: the static fallback runs only before the check. This + * test is a guard for a later path that changes r->action. + */ + + if (ctx != NULL && ctx->action == r->action) { + return ctx; + } + + n = r->action->set_headers->nelts; + + ctx = nxt_mp_zget(r->mem_pool, sizeof(nxt_http_set_headers_ctx_t)); + if (nxt_slow_path(ctx == NULL)) { + return NULL; + } + + ctx->value = nxt_mp_zalloc(r->mem_pool, sizeof(nxt_str_t) * n); + if (nxt_slow_path(ctx->value == NULL)) { + return NULL; + } + + ctx->state = nxt_mp_zalloc(r->mem_pool, n); + if (nxt_slow_path(ctx->state == NULL)) { + return NULL; + } + + ctx->action = r->action; + r->set_headers = ctx; + + return ctx; +} + + +/* + * Resolve key "i" of "response_headers" once for this request, and store + * the result in ctx. A later call returns the stored result. So a + * template, a JavaScript one too, is run only once. + * + * A null value gives a null string, which removes the field. A value that + * is not safe in a field gives NXT_DECLINED, and the key is skipped. + * nxt_http_set_headers() and nxt_http_set_headers_field_op() both use this, + * so they agree on what each key does. + */ + +static nxt_int_t +nxt_http_set_headers_value(nxt_http_request_t *r, + nxt_http_set_headers_ctx_t *ctx, nxt_uint_t i) +{ + nxt_int_t ret; + nxt_str_t *value; + nxt_router_conf_t *rtcf; + nxt_http_header_val_t *hv; + + switch (ctx->state[i]) { + + case NXT_HTTP_SET_HEADERS_RESOLVED: + return NXT_OK; + + case NXT_HTTP_SET_HEADERS_REJECTED: + return NXT_DECLINED; + + default: + break; + } + + hv = ctx->action->set_headers->elts; + hv = &hv[i]; + value = &ctx->value[i]; + + if (hv->value == NULL) { + nxt_str_null(value); + + } else if (nxt_tstr_is_const(hv->value)) { + nxt_tstr_str(hv->value, value); + + } else { + rtcf = r->conf->socket_conf->router_conf; + + ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, + &r->tstr_cache, r, r->mem_pool); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + ret = nxt_tstr_query(&r->task, r->tstr_query, hv->value, value); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + } + + if (value->start != NULL + && nxt_slow_path(!nxt_http_header_value_is_safe(value))) + { + ctx->state[i] = NXT_HTTP_SET_HEADERS_REJECTED; + return NXT_DECLINED; + } + + ctx->state[i] = NXT_HTTP_SET_HEADERS_RESOLVED; + + return NXT_OK; +} + + static nxt_http_field_t * nxt_http_resp_header_find(nxt_http_request_t *r, u_char *name, size_t length) { @@ -181,14 +408,13 @@ nxt_http_resp_header_find(nxt_http_request_t *r, u_char *name, size_t length) nxt_int_t nxt_http_set_headers(nxt_http_request_t *r) { - u_char *rejected; - nxt_int_t ret; - nxt_uint_t i, n; - nxt_str_t *value; - nxt_http_field_t *f; - nxt_router_conf_t *rtcf; - nxt_http_action_t *action; - nxt_http_header_val_t *hv, *header; + nxt_int_t ret; + nxt_uint_t i, n; + nxt_str_t *value; + nxt_http_field_t *f; + nxt_http_action_t *action; + nxt_http_header_val_t *hv, *header; + nxt_http_set_headers_ctx_t *ctx; action = r->action; @@ -200,66 +426,50 @@ nxt_http_set_headers(nxt_http_request_t *r) return NXT_OK; } - rtcf = r->conf->socket_conf->router_conf; - header = action->set_headers->elts; n = action->set_headers->nelts; - value = nxt_mp_zalloc(r->mem_pool, sizeof(nxt_str_t) * n); - if (nxt_slow_path(value == NULL)) { - return NXT_ERROR; - } + /* + * nxt_http_set_headers_field_op() can have resolved some keys already. + * Their stored results are used here, and they are not resolved again. + */ - rejected = nxt_mp_zalloc(r->mem_pool, n); - if (nxt_slow_path(rejected == NULL)) { + ctx = nxt_http_set_headers_ctx(r); + if (nxt_slow_path(ctx == NULL)) { return NXT_ERROR; } + value = ctx->value; + for (i = 0; i < n; i++) { hv = &header[i]; - if (hv->value == NULL) { - continue; - } - - if (nxt_tstr_is_const(hv->value)) { - nxt_tstr_str(hv->value, &value[i]); - - } else { - ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, - &r->tstr_cache, r, r->mem_pool); - if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; - } + ret = nxt_http_set_headers_value(r, ctx, i); - ret = nxt_tstr_query(&r->task, r->tstr_query, hv->value, &value[i]); - if (nxt_slow_path(ret != NXT_OK)) { - return NXT_ERROR; - } - } - - if (value[i].start != NULL - && nxt_slow_path(!nxt_http_header_value_is_safe(&value[i]))) - { + if (nxt_slow_path(ret == NXT_DECLINED)) { nxt_log(&r->task, NXT_LOG_INFO, "set_headers \"%V\": dropping value containing control " "bytes (HTTP response-splitting protection)", &hv->name); /* - * Mark the entry as rejected instead of clearing the value: + * The entry is marked as rejected instead of clearing the value: * a NULL value means "delete this header", and letting an * attacker-triggered rejection remove an existing response * header (e.g. an app-emitted X-Frame-Options) would fail * open. Rejected entries are skipped entirely below, so a * pre-existing same-named header survives untouched. */ - rejected[i] = 1; + continue; + } + + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; } } for (i = 0; i < n; i++) { - if (rejected[i]) { + if (ctx->state[i] == NXT_HTTP_SET_HEADERS_REJECTED) { continue; } diff --git a/test/python/content_encoding/wsgi.py b/test/python/content_encoding/wsgi.py new file mode 100644 index 000000000..0a965571e --- /dev/null +++ b/test/python/content_encoding/wsgi.py @@ -0,0 +1,15 @@ +def application(environ, start_response): + # Echo the request body, labelled as gzip: the application says that it + # coded the body itself. + content_length = int(environ.get('CONTENT_LENGTH', 0)) + body = bytes(environ['wsgi.input'].read(content_length)) + + start_response( + '200', + [ + ('Content-Type', 'text/plain'), + ('Content-Length', str(len(body))), + ('Content-Encoding', 'gzip'), + ], + ) + return [body] diff --git a/test/test_python_compression.py b/test/test_python_compression.py index 2542d8a86..c61df2260 100644 --- a/test/test_python_compression.py +++ b/test/test_python_compression.py @@ -337,3 +337,139 @@ def test_python_compression_app_timeout(encoding): time.sleep(1) assert check(2001, encoding, decode) is None, 'response after the timeout' + + +# Large enough to reach the router through shared memory, so the compressor +# takes it (see test_php_compression.py and issue 162 for small bodies). +BODY = gzip.compress(b'A' * 100000, compresslevel=0) + + +def post(body, url='/', accept='gzip', timeout=30): + # The "variables" application echoes the request body with the request's + # Content-Type. It needs a Custom-Header to build its response. Raw + # socket: the response body must not be decoded. + sock = socket.create_connection(('127.0.0.1', 8080), timeout) + sock.settimeout(timeout) + + try: + sock.sendall( + f'POST {url} HTTP/1.1\r\n' + f'Host: localhost\r\n' + f'Content-Type: text/plain\r\n' + f'Custom-Header: 557\r\n' + f'Content-Length: {len(body)}\r\n' + f'Accept-Encoding: {accept}\r\n' + f'Connection: close\r\n\r\n'.encode() + + body + ) + + chunks = [] + while True: + data = sock.recv(256 * 1024) + if not data: + break + chunks.append(data) + + finally: + sock.close() + + head, _, data = b''.join(chunks).partition(b'\r\n\r\n') + lines = head.split(b'\r\n') + + headers = {} + for line in lines[1:]: + name, _, value = line.partition(b':') + headers[name.strip().decode()] = value.strip().decode() + + if headers.get('Transfer-Encoding') == 'chunked': + data = dechunk(data) + + return int(lines[0].split()[1]), headers, data + + +def configure(response_headers, app='variables'): + client.load(app) + + action = {"pass": f"applications/{app}"} + + if response_headers is not None: + action["response_headers"] = response_headers + + assert 'success' in client.conf( + { + "http": { + "compression": { + "compressors": [{"encoding": "gzip", "level": 1}] + } + } + }, + 'settings', + ), 'compression' + assert 'success' in client.conf([{"action": action}], 'routes'), 'routes' + assert 'success' in client.conf( + {"*:8080": {"pass": "routes"}}, 'listeners' + ), 'listeners' + + +def test_python_compression_control(): + # Without a configured Content-Encoding, the echoed body is compressed. + # So the test below passes because of the configured field. + configure(None) + + status, headers, body = post(BODY) + + assert status == 200, 'status' + assert headers.get('Content-Encoding') == 'gzip', 'compressed' + assert gzip.decompress(body) == BODY, 'gzip of the echoed bytes' + + +def test_python_compression_response_headers_content_encoding(): + # https://github.com/freeunitorg/freeunit/issues/557, application path. + # The application sends bytes that are already gzip, and the action + # names their coding in "response_headers". The compressor must not + # code them again: that Content-Encoding replaces the compressor's. + configure({"Content-Encoding": "gzip"}) + + status, headers, body = post(BODY) + + assert status == 200, 'status' + assert headers.get('Content-Encoding') == 'gzip', 'the configured coding' + assert len(body) == len(BODY), 'the application length' + assert body == BODY, 'the application bytes, coded once' + + +def test_python_compression_response_headers_content_encoding_template(): + # A template value that is not safe in a field is never sent. So it + # must not keep the compressor out. Otherwise the body goes out with no + # Content-Encoding to a client that refused identity. + configure({"Content-Encoding": "$arg_c"}) + + status, headers, body = post(BODY, url='/?c=gzip') + + assert status == 200, 'status' + assert headers.get('Content-Encoding') == 'gzip', 'the configured coding' + assert body == BODY, 'the application bytes, coded once' + + status, headers, body = post( + BODY, url='/?c=gzip%0d%0a', accept='gzip, identity;q=0' + ) + + assert status == 200, 'unsafe value: status' + assert headers.get('Content-Encoding') == 'gzip', 'the compressor coding' + assert gzip.decompress(body) == BODY, 'gzip of the echoed bytes' + + +def test_python_compression_app_coding_removed_is_not_coded_again(): + # The application codes its own body and sends its own Content-Encoding. + # A null Content-Encoding in "response_headers" then removes only the + # field. The removal rule is for a coding that Unit would apply, so it + # does not apply here: the bytes are not coded again, and a client that + # refused identity gets no 406. + configure({"Content-Encoding": None}, app='content_encoding') + + for accept in ('gzip', 'gzip, identity;q=0'): + status, headers, body = post(BODY, accept=accept) + + assert status == 200, f'{accept}: status' + assert 'Content-Encoding' not in headers, f'{accept}: removed' + assert body == BODY, f'{accept}: the application bytes' diff --git a/test/test_static_compression.py b/test/test_static_compression.py index 34cbfb623..e9bd4e972 100644 --- a/test/test_static_compression.py +++ b/test/test_static_compression.py @@ -403,9 +403,10 @@ def test_static_compression_vary_merge_identity(temp_dir, configured, expected): # needs a language-module test, not a static one. -def _raw_get(url='/big.css', **headers): +def _raw_get(url='/big.css', method='GET', **headers): # Raw bytes: the body has to be decompressed, so it must not be decoded. - raw = client.get( + raw = client.http( + method, url=url, headers={'Host': 'localhost', 'Connection': 'close', **headers}, encoding='latin-1', @@ -424,6 +425,14 @@ def _raw_get(url='/big.css', **headers): return status, hdrs, body +def _age(path): + # The ETag of a file written in the current second is weak by design + # (see test_static.py). Move the mtime back, so that a weak ETag can + # come only from compression. + when = os.stat(path).st_mtime - 10 + os.utime(path, (when, when)) + + def test_static_compression_range_identity_refused(temp_dir): # A Range is served as identity -- coding a byte slice would compress the # wrong bytes -- so a client that sent "identity;q=0" must not be given @@ -886,3 +895,201 @@ def test_static_compression_malformed_weight(temp_dir): status, headers, _ = _raw_get(**{'Accept-Encoding': 'gzip;q=0.5;ext=1'}) assert status == 200 assert headers.get('Content-Encoding') == 'gzip', 'q=0.5 then an extension' + + +def _precompressed(temp_dir, response_headers): + # A precompressed copy served as "$uri.gz", the usual way to serve such + # files with Unit. No "types": the ".gz" extension has no media type. + # So every coding in "compressors" applies to it, as in issue 557. + stored = gzip.compress(b'body{color:red}' * 500) + Path(f'{temp_dir}/assets/pre.css.gz').write_bytes(stored) + _age(f'{temp_dir}/assets/pre.css.gz') + + assert 'success' in client.conf( + {"compressors": [{"encoding": "gzip", "level": 5}]}, + 'settings/http/compression', + ), 'compression without types' + + action = {"share": f'{temp_dir}/assets$uri.gz'} + + if response_headers is not None: + action["response_headers"] = response_headers + + assert 'success' in client.conf(action, 'routes/0/action'), 'action' + + return stored + + +@pytest.mark.parametrize('name', ['Content-Encoding', 'content-encoding']) +def test_static_compression_response_headers_content_encoding(temp_dir, name): + # https://github.com/freeunitorg/freeunit/issues/557: "response_headers" + # are applied after the compressor ran. Their Content-Encoding replaced + # the one the compressor added, so the stored gzip bytes went out coded + # a second time under a single "gzip", with a weakened ETag. + stored = _precompressed( + temp_dir, {"Content-Type": "text/css", name: "gzip"} + ) + + for method in ('GET', 'HEAD'): + status, headers, body = _raw_get( + url='/pre.css', method=method, **{'Accept-Encoding': 'gzip'} + ) + + assert status == 200, f'{method} status' + assert headers[name] == 'gzip', f'{method} the configured coding' + assert headers['Content-Length'] == str(len(stored)), ( + f'{method} the stored length' + ) + assert not headers['ETag'].startswith('W/'), f'{method} strong ETag' + + if method == 'GET': + assert body == stored, 'the stored bytes, coded once' + assert gzip.decompress(body) == b'body{color:red}' * 500 + + +def test_static_compression_precompressed_control(temp_dir): + # The same share without a configured Content-Encoding is still + # compressed. So the test above passes because of the configured + # field, not because this share is never compressed. + stored = _precompressed(temp_dir, {"Content-Type": "text/css"}) + + status, headers, body = _raw_get( + url='/pre.css', **{'Accept-Encoding': 'gzip'} + ) + + assert status == 200, 'status' + assert headers['Content-Encoding'] == 'gzip', 'compressed' + assert headers['ETag'].startswith('W/'), 'weak ETag for the coded copy' + assert gzip.decompress(body) == stored, 'gzip of the stored bytes' + + +def test_static_compression_response_headers_content_encoding_removed( + temp_dir, +): + # A null value removes the field. The compressor must then not run, + # or the gzip bytes go out with no Content-Encoding at all. The body is + # identity, so a client that refused identity still gets a 406. + _age(f'{temp_dir}/assets/big.css') + + assert 'success' in client.conf( + { + "share": f'{temp_dir}/assets$uri', + "response_headers": {"Content-Encoding": None}, + }, + 'routes/0/action', + ), 'configure' + + status, headers, body = _raw_get(**{'Accept-Encoding': 'gzip'}) + + assert status == 200, 'status' + assert 'Content-Encoding' not in headers, 'removed' + assert headers['Content-Length'] == '7500', 'the identity length' + assert body == b'body{color:red}' * 500, 'identity bytes' + assert not headers['ETag'].startswith('W/'), 'strong ETag' + + status, _, _ = _raw_get(**{'Accept-Encoding': 'gzip, identity;q=0'}) + + assert status == 406, 'identity refused' + + +def test_static_compression_response_headers_content_encoding_template( + temp_dir, +): + # A template value is resolved when compression is decided, with the + # same code that later sets the field. A value that is not safe in a + # field is never sent. So it must not keep the compressor out either, + # or the body goes out with no Content-Encoding to a client that + # refused identity. + stored = _precompressed( + temp_dir, {"Content-Type": "text/css", "Content-Encoding": "$arg_c"} + ) + + status, headers, body = _raw_get( + url='/pre.css?c=gzip', **{'Accept-Encoding': 'gzip'} + ) + + assert status == 200, 'status' + assert headers['Content-Encoding'] == 'gzip', 'the configured coding' + assert body == stored, 'the stored bytes, coded once' + + status, headers, body = _raw_get( + url='/pre.css?c=gzip%0d%0a', + **{'Accept-Encoding': 'gzip, identity;q=0'}, + ) + + assert status == 200, 'unsafe value: status' + assert headers.get('Content-Encoding') == 'gzip', 'the compressor coding' + assert gzip.decompress(body) == stored, 'gzip of the stored bytes' + + +def test_static_compression_response_headers_content_encoding_skipped_key( + temp_dir, +): + # Keys are applied in order, and an unsafe value skips its key. So the + # result comes from the last key that is not skipped: here the null, + # which removes the field. The body is identity, and a client that + # refused identity gets 406. + _precompressed( + temp_dir, + { + "Content-Type": "text/css", + "Content-Encoding": None, + "content-encoding": "$arg_c", + }, + ) + + status, _, _ = _raw_get( + url='/pre.css?c=gzip%0d%0a', + **{'Accept-Encoding': 'gzip, identity;q=0'}, + ) + + assert status == 406, 'identity refused' + + +def test_static_compression_response_headers_content_encoding_empty( + temp_dir, +): + # "$arg_c" with no "c" argument expands to an empty string, as "?c=" + # does. An empty string is a value, not a removal: the field is sent + # empty, nothing is compressed, and a client that refused identity gets + # no 406. + stored = _precompressed( + temp_dir, {"Content-Type": "text/css", "Content-Encoding": "$arg_c"} + ) + + for url, accept in ( + ('/pre.css', 'gzip, identity;q=0'), + ('/pre.css?c=', 'gzip'), + ): + status, headers, body = _raw_get( + url=url, **{'Accept-Encoding': accept} + ) + + assert status == 200, f'{url} status' + assert headers['Content-Encoding'] == '', f'{url} an empty field' + assert body == stored, f'{url} the stored bytes' + + +def test_static_compression_response_headers_content_encoding_resolved_once( + temp_dir, +): + # The check resolves the value, and the header gets that same value. + # "$response_header_*" is not cached, and the static handler adds + # Accept-Ranges only after the check. A second resolution at send + # would give "bytes" here. + stored = _precompressed( + temp_dir, + { + "Content-Type": "text/css", + "Content-Encoding": "$response_header_accept_ranges", + }, + ) + + status, headers, body = _raw_get( + url='/pre.css', **{'Accept-Encoding': 'gzip'} + ) + + assert status == 200, 'status' + assert headers['Accept-Ranges'] == 'bytes', 'Accept-Ranges is sent' + assert headers['Content-Encoding'] == '', 'the value of the check' + assert body == stored, 'the stored bytes' From c1dfb0471389396da608d3d7a168fc1613d25467 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 17:11:27 +0200 Subject: [PATCH 368/391] ci: let Dependabot update cargo dependencies Dependabot only watched GitHub Actions. Add the Cargo projects that have a Cargo.lock. test/requirements.txt is not added: no CI job installs it, so a raised floor would merge untested. Cargo updates come weekly. OpenTelemetry, tonic and prost crates move together in one group. A group opens one pull request per directory, so test/fake_otlp gets its own otel pull request. A second group with "applies-to: security-updates" keeps the same crates together in security updates, because a group covers version updates only by default. Other minor and patch updates of a directory share one more group. The pull request limit is 2, and it counts per directory, not for the whole block (https://github.com/dependabot/dependabot-core/issues/10395), so one run opens at most 16 pull requests. Two ignore rules apply. Wasmtime is skipped entirely. Its version is also pinned in pkg/contrib/src/wasmtime/version, EOL.md and pkg/eol.json, which a Dependabot pull request cannot change, so it is updated by hand and stays on the 48 line. This also stops Dependabot security pull requests for Wasmtime; the weekly audit.yml run reports its advisories instead. bindgen minor and major versions are skipped. bindgen 0.68 produced empty structs under clang 22, and 0.72 is the version that was tested. See https://github.com/freeunitorg/freeunit/blob/master/src/wasm-wasi-component/Cargo.toml Not covered, on purpose: the generated pkg/docker/Dockerfile.* files, the Docker files that use the floating tag rust:slim-trixie, the and the vendored Wasmtime source tarball in pkg/contrib. The two test/wasm_component fixtures are covered: the tests build a copy of each with "cargo component build", which writes src/bindings.rs again, and the exports do not come from that file. Co-Authored-By: Claude Sonnet 5.5 Reviewed-by: DeepSeek Reviewed-by: Codex --- .github/dependabot.yml | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c75e875a0..c7a01a924 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,62 @@ updates: directory: "/" schedule: interval: weekly + + # Every Cargo.toml that has its own Cargo.lock. tools/unitctl is one + # workspace, so its root covers the three member crates. The tests build + # a copy of each test/wasm_component fixture with "cargo component build", + # which writes src/bindings.rs again. The exports come from + # wasi::http::proxy::export!, so a bump cannot break a fixture through + # that file. + - package-ecosystem: cargo + directories: + - "/tools/unitctl" + - "/src/otel" + - "/src/wasm-wasi-component" + - "/pkg/eol" + - "/test/fake_otlp" + - "/test/fake_upstream" + - "/test/wasm_component/hello_world" + - "/test/wasm_component/spin_loop" + schedule: + interval: weekly + # The limit counts per directory, not for the whole block + # (https://github.com/dependabot/dependabot-core/issues/10395). With the + # groups below, a directory needs one or two pull requests a week. + open-pull-requests-limit: 2 + commit-message: + prefix: "build(deps)" + # A group opens one pull request per directory, so test/fake_otlp gets + # its own otel pull request. "group-by: dependency-name" would join the + # directories but split the otel crates, which must move together. + groups: + otel: + patterns: + - "opentelemetry*" + - "tonic*" + - "prost*" + # A group covers version updates only, so security updates need their + # own group to keep the otel crates together. + otel-security: + applies-to: security-updates + patterns: + - "opentelemetry*" + - "tonic*" + - "prost*" + minor-and-patch: + update-types: + - "minor" + - "patch" + ignore: + # Wasmtime is updated by hand. Its version is also pinned in + # pkg/contrib/src/wasmtime/version, EOL.md and pkg/eol.json, which a + # Dependabot pull request cannot change. It stays on the 48 LTS line. + # This also stops Dependabot security pull requests for Wasmtime; the + # weekly audit.yml run reports its advisories instead. + - dependency-name: "wasmtime*" + # bindgen 0.68 emitted opaque structs under clang 22; 0.72 is the + # tested version. A 0.x minor can break, so bump it by hand. + - dependency-name: "bindgen" + update-types: + - "version-update:semver-minor" + - "version-update:semver-major" From 3dbfb157948bfcff3f1ccfa6c427d26f0deb189e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 17:25:50 +0200 Subject: [PATCH 369/391] java: update tomcat jars to 9.0.122 Apache Tomcat 9.0.122 fixes CVE-2026-87022, per https://tomcat.apache.org/security-9.html#Fixed_in_Apache_Tomcat_9.0.122: "WebSocket message smuggling with per-message-deflate". The fix is https://github.com/apache/tomcat/commit/959a52a227cc35101b92dae35b722546167594d6 and it is in the 9.0.122 tag. The release fixes eleven other CVEs. None of the twelve fixes is in a jar that the build downloads. FreeUnit has its own copy of the vulnerable code in src/java/nginx/unit/websocket/PerMessageDeflate.java, but no received frame reaches it. The server side never installs the transformation, does not answer with Sec-WebSocket-Extensions, and passes received frames to the application without decompression. The client side fails with a NullPointerException in the WsSession constructor before it creates the frame reader. So this commit does not port the fix. Whoever makes the client's connectToServer() work must port https://github.com/apache/tomcat/commit/959a52a227cc35101b92dae35b722546167594d6 first, because that change makes the copied code reachable. 9.0.122 is the newest 9.0.x on Maven Central (https://repo1.maven.org/maven2/org/apache/tomcat/tomcat-servlet-api/maven-metadata.xml). Each new SHA512 was checked against the .sha512 file that Maven Central publishes for the same jar. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 7 +++++++ auto/modules/java | 2 +- auto/modules/java_jar.sha512 | 18 +++++++++--------- docs/changes.xml | 11 +++++++++++ test/unit/applications/lang/java.py | 2 +- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/CHANGES b/CHANGES index 5df246419..81c88a3be 100644 --- a/CHANGES +++ b/CHANGES @@ -981,6 +981,13 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 "script" that resolves to "root" itself is also rejected; it was served as an empty 200 response. The bug had appeared in 0.4. + *) Change: the Java module now bundles Apache Tomcat 9.0.122 (from + 9.0.121). The upstream release fixes CVE-2026-87022 (WebSocket + message smuggling with per-message-deflate) and eleven other CVEs. + None of the twelve fixes is in a jar that FreeUnit bundles. The + FreeUnit WebSocket code is derived from Tomcat, but it does not + decompress received frames, so CVE-2026-87022 does not affect it. + Changes with FreeUnit 1.36.1 28 Aug 2026 *) Feature: configuration validation errors now say where they are. A diff --git a/auto/modules/java b/auto/modules/java index e6d7e3448..c74a337dc 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -238,7 +238,7 @@ cat << END > $NXT_JAVA_JARS static const char *nxt_java_system_jars[] = { END -NXT_TOMCAT_VERSION=9.0.121 +NXT_TOMCAT_VERSION=9.0.122 NXT_JAR_VERSION=$NXT_TOMCAT_VERSION diff --git a/auto/modules/java_jar.sha512 b/auto/modules/java_jar.sha512 index 421ea5350..c4de2261a 100644 --- a/auto/modules/java_jar.sha512 +++ b/auto/modules/java_jar.sha512 @@ -3,12 +3,12 @@ ab441acf5551a7dc81c353eaccb3b3df9e89a48987294d19e39acdb83a5b640fcdff7414cee29f5b 609516a5c3b7fa9e256b768124d3c822f28ce426687a479eb57b0f2f6dc82ee6c3560f8e790394b71feebc409eda78a04d6c03254bb41127506ffcf6b7905a61 jetty-http-9.4.58.v20250814.jar 5f2b1efe5f6279ed9c3a25d8c3a6413b195a6957500752cb7b59fc8e8bc43d01eeb2e97b619bfb74ff5ec96cd8893ba1d9d372f8a226ff197d3b452febafa146 jetty-server-9.4.58.v20250814.jar c4505826e68be8d8736eba5cd29fd86ada455ee5765f8d46d33085b788eb3295023b167354178441e80c97edde9f988f89bf3959c59ef8aada56e5d863673021 jetty-util-9.4.58.v20250814.jar -173325a3e237f1f99fb05bbf8569040fb93af0ebadfb73019aaa9640fd351b16cc66a9b69deed8f9f3a945d40413b7ca9633fef7d877778436b341452b0ad653 tomcat-api-9.0.121.jar -aed5789fd28040f5bc88336a3fef1372d6949e2fc912381c7a32594514f18250e7b45f671260932ea04d472675d5f2f35010ef2109dce0416b0706ffc0c701bc tomcat-el-api-9.0.121.jar -3464120da470ac4fd75dad030aa58d5e9a07c36e34287b9febdf5f2025e7642dc43f874095dc0379968a469ffedfef67ae0568ca611d29506fc6175858c5ca76 tomcat-jasper-9.0.121.jar -78d0ec8c83d8733660f62d4ebbb037536cf242fdf3d1111c32a26ecd5100022bd800cd791e0e510cb383119c0bf068b9bf2eaff8cf7f8d5caa0a5c93af61a32c tomcat-jasper-el-9.0.121.jar -88970a8c75f623d157cef417dd0ba0d2ce1000be56c7f205545c087f46c1a9130c54dba49e6926d526a7d74f947d13d020df882fe54c2700385a43e3ca6e7e43 tomcat-jsp-api-9.0.121.jar -19f3e032b914aed600199f5a2e95cf20374daf24fef999586d4710fce54ffb124f8ffadd314c337c327db18bcbfa06c87862b3dc7606a465388aba9208ebca48 tomcat-juli-9.0.121.jar -b74470d50cc6c058cb0a2b28e2b7780f731003fc9d0ad9ea8e53a145ff548ace2b6b952baecc15cf4fd052bec4b80730a9d7c33df721e0fa67998beffeeea790 tomcat-servlet-api-9.0.121.jar -f39189d511dee85c766f5046f53c17e6ff541cbec2d325adf2ebf043997fa29212df1b53effd029a72acab5bb37f130a25a949e5e48983744f862907ce959af1 tomcat-util-9.0.121.jar -a272f0d5dde67ef5468ac7fb8e9b33fb5457b999952931a6329ea767a91001792fd86f2cd358a6152dadba8585f40016519d1cdc135d250f987a1a6d2997b026 tomcat-util-scan-9.0.121.jar +36383f02b50b0243a78a71500bbd885ec699b429fa2d1d8e1723276d4aa525e0e4fc1cb7dc0b977e48d182dd02feb6fd4db29e6af348cd274c1a10a10ebb8f39 tomcat-api-9.0.122.jar +bc5890603c5fa3e4dd817fd6fcf7d196a01a9c30ddab35f276d15239b5efd1d74df3ee3468dc3a2cfd8c366e7dcc26361f400a47faf32486c4453e53e153c2be tomcat-el-api-9.0.122.jar +20aba1db626aff6b66446721ae5a8a99c8040e285da01a4e0c649841957304bef43538939f65a98749be0cfc2e1ea3a000c3b733948fbf9a081f616fdb51a7e4 tomcat-jasper-9.0.122.jar +94f731d2e044a17c41483853f43c07ab48be45effb87678fd8f0cf6d5b6ff48b8f0482349ccb5b47c38e586577e51695727c97287cadb34f49e3c6ae851715bc tomcat-jasper-el-9.0.122.jar +188126266f4d1db7ebe2a5ca48ce0abb4b7ea75c53c79a18a117bfac8afecb43324fdbe933459239989533142d8d246bf2798545e63b638c8955b7de5dc8178b tomcat-jsp-api-9.0.122.jar +f7a756e6e91d378d8da41dc906372b8f3dcd2a8e985393eff6a79ea94d7b31e631b9ebe5661681f5cfd5072362d0097e6b15c8444c87f66af13efe08e78ee82d tomcat-juli-9.0.122.jar +f81255c4d696ef156c82e5b6ea543105712327e8ebcd95e33171f7d4cf5d380ce86f88350e08054ba6743675e76760915bb6fd20d6d72d34ea8d2c72b6dc7d43 tomcat-servlet-api-9.0.122.jar +78f5cb17044f8ed3a837756d6600048eb326a00512a9ef45bbd9bbd0840e51234cb18847af4e55c0545e412ac9b6a736ab113f6efa0096ea5b2843c2d8ed5bf9 tomcat-util-9.0.122.jar +94aa42c471809a543fe99017f3ad873956cf3fc00fb17f34e7e37950b7948e784bb68d7e3fb005190c6d0db1eba85be60d7db2c70ee09e19bd1f50744b35978f tomcat-util-scan-9.0.122.jar diff --git a/docs/changes.xml b/docs/changes.xml index 359742979..374eec5c9 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -1313,6 +1313,17 @@ rejected; it was served as an empty 200 response. The bug had appeared in 0.4. + + +the Java module now bundles Apache Tomcat 9.0.122 (from 9.0.121). The +upstream release fixes CVE-2026-87022 (WebSocket message smuggling with +per-message-deflate) and eleven other CVEs. None of the twelve fixes is in a +jar that FreeUnit bundles. The FreeUnit WebSocket code is derived from Tomcat, +but it does not decompress received frames, so CVE-2026-87022 does not affect +it. + + + diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index 25258c01d..5fcabcc76 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -53,7 +53,7 @@ def prepare_env(self, script): os.makedirs(classes_path) classpath = ( - f'{option.current_dir}/build/tomcat-servlet-api-9.0.121.jar' + f'{option.current_dir}/build/tomcat-servlet-api-9.0.122.jar' ) ws_jars = glob.glob( From b5bf3e6768327644bfa457a5c412b9ef8b5e7be3 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 17:55:04 +0200 Subject: [PATCH 370/391] docker: add Go 1.27, Perl 5.44 and Java 27, and upgrade base packages Add the runtimes that upstream now publishes: - golang:1.27-trixie (Go 1.27 was released on 2026-08-19). - perl:5.44-trixie (Perl 5.44 was released on 2026-07-15). - eclipse-temurin:27-jdk-resolute (Java 27 was released on 2026-09-22). The EOL dates come from https://endoflife.date (go, perl, eclipse-temurin). Go 1.27 has no published EOL yet. Its date, 2027-08, follows the same rule as Go 1.26: support ends at the second release after it. Eclipse Temurin has no 27-jdk-noble tag. Its only Ubuntu tag for Java 27 is 27-jdk-resolute (Ubuntu 26.04). VARIANT_java moves from noble to resolute, so the Java 17, 21, 25 and 26 images move to Ubuntu 26.04 too. Each of them has a -jdk-resolute tag for amd64 and arm64. Java 26 passed its upstream EOL on 2026-09-22. It gets the "EOL" note, as Go 1.25, Node.js 20 and Perl 5.38 have. The four images stay in the matrix for the 12-month grace period of pkg/eol.json. The template ran no package upgrade. An image whose base is no longer rebuilt upstream, such as node:20, kept the Debian packages of its last base build. The template now runs "apt-get upgrade -y" after the first "apt-get update", in the same RUN layer. The layer already ends with "rm -rf /var/lib/apt/lists/*". The runtime stages of the three pkg/docker/local/ Dockerfiles get the same step. The Dockerfiles are regenerated with "make -C pkg/docker dockerfiles". The CI matrices in build-test.yml and release-docker.yml read pkg/eol.json, so they pick up the new versions with no workflow change. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 8 ++ EOL.md | 5 +- README.md | 1 + docs/changes.xml | 16 ++++ pkg/docker/Dockerfile.go-1.25 | 1 + pkg/docker/Dockerfile.go-1.26 | 1 + pkg/docker/Dockerfile.go-1.27 | 116 +++++++++++++++++++++++++ pkg/docker/Dockerfile.java-17 | 3 +- pkg/docker/Dockerfile.java-21 | 3 +- pkg/docker/Dockerfile.java-25 | 3 +- pkg/docker/Dockerfile.java-26 | 3 +- pkg/docker/Dockerfile.java-27 | 116 +++++++++++++++++++++++++ pkg/docker/Dockerfile.minimal | 1 + pkg/docker/Dockerfile.node-20 | 1 + pkg/docker/Dockerfile.node-22 | 1 + pkg/docker/Dockerfile.node-24 | 1 + pkg/docker/Dockerfile.node-26 | 1 + pkg/docker/Dockerfile.perl-5.38 | 1 + pkg/docker/Dockerfile.perl-5.40 | 1 + pkg/docker/Dockerfile.perl-5.42 | 1 + pkg/docker/Dockerfile.perl-5.44 | 116 +++++++++++++++++++++++++ pkg/docker/Dockerfile.php-8.3 | 1 + pkg/docker/Dockerfile.php-8.4 | 1 + pkg/docker/Dockerfile.php-8.5 | 1 + pkg/docker/Dockerfile.python-3.12 | 1 + pkg/docker/Dockerfile.python-3.12-slim | 1 + pkg/docker/Dockerfile.python-3.13 | 1 + pkg/docker/Dockerfile.python-3.13-slim | 1 + pkg/docker/Dockerfile.python-3.14 | 1 + pkg/docker/Dockerfile.python-3.14-slim | 1 + pkg/docker/Dockerfile.ruby-3.3 | 1 + pkg/docker/Dockerfile.ruby-3.4 | 1 + pkg/docker/Dockerfile.ruby-4.0 | 1 + pkg/docker/Dockerfile.wasm | 1 + pkg/docker/Makefile | 2 +- pkg/docker/README.md | 9 +- pkg/docker/local/Dockerfile.minimal | 1 + pkg/docker/local/Dockerfile.php-8.5 | 1 + pkg/docker/local/Dockerfile.wasm | 1 + pkg/docker/template.Dockerfile | 1 + pkg/eol.json | 9 +- 41 files changed, 426 insertions(+), 11 deletions(-) create mode 100644 pkg/docker/Dockerfile.go-1.27 create mode 100644 pkg/docker/Dockerfile.java-27 create mode 100644 pkg/docker/Dockerfile.perl-5.44 diff --git a/CHANGES b/CHANGES index 81c88a3be..4b503f14d 100644 --- a/CHANGES +++ b/CHANGES @@ -988,6 +988,14 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 FreeUnit WebSocket code is derived from Tomcat, but it does not decompress received frames, so CVE-2026-87022 does not affect it. + *) Feature: Docker images for Go 1.27, Perl 5.44 and Java 27. The Java + images are now based on Ubuntu 26.04 (resolute) instead of 24.04 + (noble), because Eclipse Temurin publishes Java 27 only for 26.04. + + *) Change: the Docker images now install the pending updates of their + base image when they are built. Before, an image kept the packages + of its base image as they were when that image was published. + Changes with FreeUnit 1.36.1 28 Aug 2026 *) Feature: configuration validation errors now say where they are. A diff --git a/EOL.md b/EOL.md index 950aa7de8..01c29372c 100644 --- a/EOL.md +++ b/EOL.md @@ -13,10 +13,12 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Go | 1.24 (EOL) † | Feb 2026 | Feb 2027 | | Go | 1.25 (EOL) † | Aug 2026 | Aug 2027 | | Go | 1.26 | Feb 2027 | Feb 2028 | +| Go | 1.27 | Aug 2027 | Aug 2028 | | Java (JSC) | 17 (LTS) | Oct 2027 | Oct 2028 | | Java (JSC) | 21 (LTS) | Dec 2029 | Dec 2030 | | Java (JSC) | 25 (LTS) | Sep 2031 | Sep 2032 | -| Java (JSC) | 26 | Sep 2026 | Sep 2027 | +| Java (JSC) | 26 (EOL) † | Sep 2026 | Sep 2027 | +| Java (JSC) | 27 | Mar 2027 | Mar 2028 | | Node.js | 20 (LTS) (EOL) † | Apr 2026 | Apr 2027 | | Node.js | 22 (LTS) | Apr 2027 | Apr 2028 | | Node.js | 24 (LTS) | Apr 2028 | Apr 2029 | @@ -24,6 +26,7 @@ EOL dates are tracked at [endoflife.date](https://endoflife.date). | Perl | 5.38 (EOL) † | Jul 2026 | Jul 2027 | | Perl | 5.40 | Jun 2027 | Jun 2028 | | Perl | 5.42 | Jul 2028 | Jul 2029 | +| Perl | 5.44 | Jul 2029 | Jul 2030 | | PHP | 8.3 | Dec 2027 | Dec 2028 | | PHP | 8.4 | Dec 2028 | Dec 2029 | | PHP | 8.5 | Dec 2029 | Dec 2030 | diff --git a/README.md b/README.md index 9f3111518..05c6e6fee 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ and are available for `linux/amd64` and `linux/arm64`. | Node.js 24 | `ghcr.io/freeunitorg/freeunit:latest-node-24` | | Node.js 22 | `ghcr.io/freeunitorg/freeunit:latest-node-22` | | Node.js 20 | `ghcr.io/freeunitorg/freeunit:latest-node-20` | +| Go 1.27 | `ghcr.io/freeunitorg/freeunit:latest-go-1.27` | | Go 1.26 | `ghcr.io/freeunitorg/freeunit:latest-go-1.26` | | Go 1.25 | `ghcr.io/freeunitorg/freeunit:latest-go-1.25` | | Ruby 4.0 | `ghcr.io/freeunitorg/freeunit:latest-ruby-4.0` | diff --git a/docs/changes.xml b/docs/changes.xml index 374eec5c9..8dfdaaf84 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -1324,6 +1324,22 @@ it. + + +Docker images for Go 1.27, Perl 5.44 and Java 27. The Java images are now +based on Ubuntu 26.04 (resolute) instead of 24.04 (noble), because Eclipse +Temurin publishes Java 27 only for 26.04. + + + + + +the Docker images now install the pending updates of their base image when +they are built. Before, an image kept the packages of its base image as they +were when that image was published. + + + diff --git a/pkg/docker/Dockerfile.go-1.25 b/pkg/docker/Dockerfile.go-1.25 index 93ed5cac6..95862cb6b 100644 --- a/pkg/docker/Dockerfile.go-1.25 +++ b/pkg/docker/Dockerfile.go-1.25 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.go-1.26 b/pkg/docker/Dockerfile.go-1.26 index e13cad5a1..57979bd29 100644 --- a/pkg/docker/Dockerfile.go-1.26 +++ b/pkg/docker/Dockerfile.go-1.26 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.go-1.27 b/pkg/docker/Dockerfile.go-1.27 new file mode 100644 index 000000000..ff866d2a6 --- /dev/null +++ b/pkg/docker/Dockerfile.go-1.27 @@ -0,0 +1,116 @@ +FROM golang:1.27-trixie + +LABEL org.opencontainers.image.title="FreeUnit (go-1.27)" +LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." +LABEL org.opencontainers.image.url="https://freeunit.org" +LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" +LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" +LABEL org.opencontainers.image.vendor="FreeUnit Community " +LABEL org.opencontainers.image.version="1.36.1" + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ + && export RUST_VERSION=1.95.0 \ + && export RUSTUP_HOME=/usr/src/unit/rustup \ + && export CARGO_HOME=/usr/src/unit/cargo \ + && export PATH=/usr/src/unit/cargo/bin:$PATH \ + && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) rustArch="x86_64-unknown-linux-gnu"; rustupSha256="6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d" ;; \ + arm64) rustArch="aarch64-unknown-linux-gnu"; rustupSha256="1cffbf51e63e634c746f741de50649bbbcbd9dbe1de363c9ecef64e278dba2b2" ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac \ + && url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" \ + && curl -L -O "$url" \ + && echo "${rustupSha256} *rustup-init" | sha256sum -c - \ + && chmod +x rustup-init \ + && ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch} \ + && rm rustup-init \ + && rustup --version \ + && cargo --version \ + && rustc --version \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && mkdir -p /usr/src/unit \ + && cd /usr/src/unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && cd unit \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-fPIC" dpkg-buildflags --get CFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && CONFIGURE_ARGS_MODULES="--prefix=/usr \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --runstatedir=/var/run \ + --pid=/var/run/unit.pid \ + --logdir=/var/log \ + --log=/var/log/unit.log \ + --tmpdir=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ + --njs \ + --otel \ + --zlib \ + --zstd \ + --brotli" \ + && make -j $NCPU -C pkg/contrib .njs \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd \ + && make clean \ + && /bin/true \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && ./configure go --go-path=$GOPATH \ + && make -j $NCPU go-install-src libunit-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/modules \ + && ./configure go --go-path=$GOPATH \ + && make -j $NCPU go-install-src libunit-install \ + && cd \ + && rm -rf /usr/src/unit \ + && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ + [ -f "$f" ] || continue; \ + ldd "$f" | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $(realpath $n 2>/dev/null || echo $n) 2>/dev/null; done | grep -v '^diversion' | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + && /bin/true \ + && mkdir -p /var/lib/unit/ \ + && mkdir -p /docker-entrypoint.d/ \ + && groupadd --gid 999 unit \ + && useradd \ + --uid 999 \ + --gid unit \ + --no-create-home \ + --home /nonexistent \ + --comment "unit user" \ + --shell /bin/false \ + unit \ + && apt-get update \ + && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stderr /var/log/unit.log + +COPY docker-entrypoint.sh /usr/local/bin/ +COPY welcome.* /usr/share/unit/welcome/ +COPY seccomp-no-af-alg.json /usr/share/unit/ + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +EXPOSE 80 +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.java-17 b/pkg/docker/Dockerfile.java-17 index bd1919ae7..fae252503 100644 --- a/pkg/docker/Dockerfile.java-17 +++ b/pkg/docker/Dockerfile.java-17 @@ -1,4 +1,4 @@ -FROM eclipse-temurin:17-jdk-noble +FROM eclipse-temurin:17-jdk-resolute LABEL org.opencontainers.image.title="FreeUnit (java-17)" LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.java-21 b/pkg/docker/Dockerfile.java-21 index 212cc152a..7a5eadc31 100644 --- a/pkg/docker/Dockerfile.java-21 +++ b/pkg/docker/Dockerfile.java-21 @@ -1,4 +1,4 @@ -FROM eclipse-temurin:21-jdk-noble +FROM eclipse-temurin:21-jdk-resolute LABEL org.opencontainers.image.title="FreeUnit (java-21)" LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.java-25 b/pkg/docker/Dockerfile.java-25 index 79393f4d5..ea196a7a6 100644 --- a/pkg/docker/Dockerfile.java-25 +++ b/pkg/docker/Dockerfile.java-25 @@ -1,4 +1,4 @@ -FROM eclipse-temurin:25-jdk-noble +FROM eclipse-temurin:25-jdk-resolute LABEL org.opencontainers.image.title="FreeUnit (java-25)" LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.java-26 b/pkg/docker/Dockerfile.java-26 index 717f10d7c..c9dbdef68 100644 --- a/pkg/docker/Dockerfile.java-26 +++ b/pkg/docker/Dockerfile.java-26 @@ -1,4 +1,4 @@ -FROM eclipse-temurin:26-jdk-noble +FROM eclipse-temurin:26-jdk-resolute LABEL org.opencontainers.image.title="FreeUnit (java-26)" LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.java-27 b/pkg/docker/Dockerfile.java-27 new file mode 100644 index 000000000..b82003368 --- /dev/null +++ b/pkg/docker/Dockerfile.java-27 @@ -0,0 +1,116 @@ +FROM eclipse-temurin:27-jdk-resolute + +LABEL org.opencontainers.image.title="FreeUnit (java-27)" +LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." +LABEL org.opencontainers.image.url="https://freeunit.org" +LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" +LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" +LABEL org.opencontainers.image.vendor="FreeUnit Community " +LABEL org.opencontainers.image.version="1.36.1" + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ + && export RUST_VERSION=1.95.0 \ + && export RUSTUP_HOME=/usr/src/unit/rustup \ + && export CARGO_HOME=/usr/src/unit/cargo \ + && export PATH=/usr/src/unit/cargo/bin:$PATH \ + && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) rustArch="x86_64-unknown-linux-gnu"; rustupSha256="6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d" ;; \ + arm64) rustArch="aarch64-unknown-linux-gnu"; rustupSha256="1cffbf51e63e634c746f741de50649bbbcbd9dbe1de363c9ecef64e278dba2b2" ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac \ + && url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" \ + && curl -L -O "$url" \ + && echo "${rustupSha256} *rustup-init" | sha256sum -c - \ + && chmod +x rustup-init \ + && ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch} \ + && rm rustup-init \ + && rustup --version \ + && cargo --version \ + && rustc --version \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && mkdir -p /usr/src/unit \ + && cd /usr/src/unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && cd unit \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-fPIC" dpkg-buildflags --get CFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && CONFIGURE_ARGS_MODULES="--prefix=/usr \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --runstatedir=/var/run \ + --pid=/var/run/unit.pid \ + --logdir=/var/log \ + --log=/var/log/unit.log \ + --tmpdir=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ + --njs \ + --otel \ + --zlib \ + --zstd \ + --brotli" \ + && make -j $NCPU -C pkg/contrib .njs \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd \ + && make clean \ + && /bin/true \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && ./configure java --jars=/usr/share/unit-jsc-common/ \ + && make -j $NCPU java-shared-install java-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/modules \ + && ./configure java --jars=/usr/share/unit-jsc-common/ \ + && make -j $NCPU java-shared-install java-install \ + && cd \ + && rm -rf /usr/src/unit \ + && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ + [ -f "$f" ] || continue; \ + ldd "$f" | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $(realpath $n 2>/dev/null || echo $n) 2>/dev/null; done | grep -v '^diversion' | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + && rm -rf /root/.m2 \ + && mkdir -p /var/lib/unit/ \ + && mkdir -p /docker-entrypoint.d/ \ + && groupadd --gid 999 unit \ + && useradd \ + --uid 999 \ + --gid unit \ + --no-create-home \ + --home /nonexistent \ + --comment "unit user" \ + --shell /bin/false \ + unit \ + && apt-get update \ + && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stderr /var/log/unit.log + +COPY docker-entrypoint.sh /usr/local/bin/ +COPY welcome.* /usr/share/unit/welcome/ +COPY seccomp-no-af-alg.json /usr/share/unit/ + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +EXPOSE 80 +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index f8e5ae895..4c7a3fb72 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.node-20 b/pkg/docker/Dockerfile.node-20 index fccba2982..ef505a8e7 100644 --- a/pkg/docker/Dockerfile.node-20 +++ b/pkg/docker/Dockerfile.node-20 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.node-22 b/pkg/docker/Dockerfile.node-22 index 8f0c640e0..87cc48677 100644 --- a/pkg/docker/Dockerfile.node-22 +++ b/pkg/docker/Dockerfile.node-22 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.node-24 b/pkg/docker/Dockerfile.node-24 index e5ab23cb7..56fce9ae4 100644 --- a/pkg/docker/Dockerfile.node-24 +++ b/pkg/docker/Dockerfile.node-24 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.node-26 b/pkg/docker/Dockerfile.node-26 index 0614cb299..c38481b4c 100644 --- a/pkg/docker/Dockerfile.node-26 +++ b/pkg/docker/Dockerfile.node-26 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.perl-5.38 b/pkg/docker/Dockerfile.perl-5.38 index 482a4820c..3a11ca109 100644 --- a/pkg/docker/Dockerfile.perl-5.38 +++ b/pkg/docker/Dockerfile.perl-5.38 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.perl-5.40 b/pkg/docker/Dockerfile.perl-5.40 index b0333fe37..ed4a20cc7 100644 --- a/pkg/docker/Dockerfile.perl-5.40 +++ b/pkg/docker/Dockerfile.perl-5.40 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.perl-5.42 b/pkg/docker/Dockerfile.perl-5.42 index 4fa2b6063..66222a286 100644 --- a/pkg/docker/Dockerfile.perl-5.42 +++ b/pkg/docker/Dockerfile.perl-5.42 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.perl-5.44 b/pkg/docker/Dockerfile.perl-5.44 new file mode 100644 index 000000000..0b966945d --- /dev/null +++ b/pkg/docker/Dockerfile.perl-5.44 @@ -0,0 +1,116 @@ +FROM perl:5.44-trixie + +LABEL org.opencontainers.image.title="FreeUnit (perl-5.44)" +LABEL org.opencontainers.image.description="Community build of FreeUnit for Docker." +LABEL org.opencontainers.image.url="https://freeunit.org" +LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" +LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" +LABEL org.opencontainers.image.vendor="FreeUnit Community " +LABEL org.opencontainers.image.version="1.36.1" + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ + && export RUST_VERSION=1.95.0 \ + && export RUSTUP_HOME=/usr/src/unit/rustup \ + && export CARGO_HOME=/usr/src/unit/cargo \ + && export PATH=/usr/src/unit/cargo/bin:$PATH \ + && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) rustArch="x86_64-unknown-linux-gnu"; rustupSha256="6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d" ;; \ + arm64) rustArch="aarch64-unknown-linux-gnu"; rustupSha256="1cffbf51e63e634c746f741de50649bbbcbd9dbe1de363c9ecef64e278dba2b2" ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac \ + && url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" \ + && curl -L -O "$url" \ + && echo "${rustupSha256} *rustup-init" | sha256sum -c - \ + && chmod +x rustup-init \ + && ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch} \ + && rm rustup-init \ + && rustup --version \ + && cargo --version \ + && rustc --version \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && mkdir -p /usr/src/unit \ + && cd /usr/src/unit \ + && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && cd unit \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-fPIC" dpkg-buildflags --get CFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS | tr ' ' '\n' | grep -v '^-specs=' | tr '\n' ' ')" \ + && CONFIGURE_ARGS_MODULES="--prefix=/usr \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --runstatedir=/var/run \ + --pid=/var/run/unit.pid \ + --logdir=/var/log \ + --log=/var/log/unit.log \ + --tmpdir=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ + --njs \ + --otel \ + --zlib \ + --zstd \ + --brotli" \ + && make -j $NCPU -C pkg/contrib .njs \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd \ + && make clean \ + && /bin/true \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && ./configure perl \ + && make -j $NCPU perl-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/modules \ + && ./configure perl \ + && make -j $NCPU perl-install \ + && cd \ + && rm -rf /usr/src/unit \ + && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ + [ -f "$f" ] || continue; \ + ldd "$f" | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $(realpath $n 2>/dev/null || echo $n) 2>/dev/null; done | grep -v '^diversion' | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + && /bin/true \ + && mkdir -p /var/lib/unit/ \ + && mkdir -p /docker-entrypoint.d/ \ + && groupadd --gid 999 unit \ + && useradd \ + --uid 999 \ + --gid unit \ + --no-create-home \ + --home /nonexistent \ + --comment "unit user" \ + --shell /bin/false \ + unit \ + && apt-get update \ + && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stderr /var/log/unit.log + +COPY docker-entrypoint.sh /usr/local/bin/ +COPY welcome.* /usr/share/unit/welcome/ +COPY seccomp-no-af-alg.json /usr/share/unit/ + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +EXPOSE 80 +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.php-8.3 b/pkg/docker/Dockerfile.php-8.3 index f80513dde..f5070d033 100644 --- a/pkg/docker/Dockerfile.php-8.3 +++ b/pkg/docker/Dockerfile.php-8.3 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.php-8.4 b/pkg/docker/Dockerfile.php-8.4 index ba5405ed0..677965517 100644 --- a/pkg/docker/Dockerfile.php-8.4 +++ b/pkg/docker/Dockerfile.php-8.4 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.php-8.5 b/pkg/docker/Dockerfile.php-8.5 index 6fe01913f..441f54b13 100644 --- a/pkg/docker/Dockerfile.php-8.5 +++ b/pkg/docker/Dockerfile.php-8.5 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.python-3.12 b/pkg/docker/Dockerfile.python-3.12 index 6d8a451f0..7c3275b1c 100644 --- a/pkg/docker/Dockerfile.python-3.12 +++ b/pkg/docker/Dockerfile.python-3.12 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.python-3.12-slim b/pkg/docker/Dockerfile.python-3.12-slim index c5ce4f34d..d4cb2e4d7 100644 --- a/pkg/docker/Dockerfile.python-3.12-slim +++ b/pkg/docker/Dockerfile.python-3.12-slim @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.python-3.13 b/pkg/docker/Dockerfile.python-3.13 index 6d94e94ed..99122ead0 100644 --- a/pkg/docker/Dockerfile.python-3.13 +++ b/pkg/docker/Dockerfile.python-3.13 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.python-3.13-slim b/pkg/docker/Dockerfile.python-3.13-slim index 3a037f41b..7a883d2af 100644 --- a/pkg/docker/Dockerfile.python-3.13-slim +++ b/pkg/docker/Dockerfile.python-3.13-slim @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.python-3.14 b/pkg/docker/Dockerfile.python-3.14 index 1d4fe0597..78daa290b 100644 --- a/pkg/docker/Dockerfile.python-3.14 +++ b/pkg/docker/Dockerfile.python-3.14 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.python-3.14-slim b/pkg/docker/Dockerfile.python-3.14-slim index 1b3a8b3e8..f829addea 100644 --- a/pkg/docker/Dockerfile.python-3.14-slim +++ b/pkg/docker/Dockerfile.python-3.14-slim @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.ruby-3.3 b/pkg/docker/Dockerfile.ruby-3.3 index 8cd08db71..991716652 100644 --- a/pkg/docker/Dockerfile.ruby-3.3 +++ b/pkg/docker/Dockerfile.ruby-3.3 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.ruby-3.4 b/pkg/docker/Dockerfile.ruby-3.4 index 5d0e5dc9c..7f7e82157 100644 --- a/pkg/docker/Dockerfile.ruby-3.4 +++ b/pkg/docker/Dockerfile.ruby-3.4 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.ruby-4.0 b/pkg/docker/Dockerfile.ruby-4.0 index b7c3ec38c..f22d77dd3 100644 --- a/pkg/docker/Dockerfile.ruby-4.0 +++ b/pkg/docker/Dockerfile.ruby-4.0 @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Dockerfile.wasm b/pkg/docker/Dockerfile.wasm index 1059c33d3..a867fa52b 100644 --- a/pkg/docker/Dockerfile.wasm +++ b/pkg/docker/Dockerfile.wasm @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="1.36.1" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/docker/Makefile b/pkg/docker/Makefile index 0d45fccf4..ef0915ea4 100644 --- a/pkg/docker/Makefile +++ b/pkg/docker/Makefile @@ -37,7 +37,7 @@ RUN_go ?= /bin/true MODULE_PREBUILD_go ?= /bin/true VERSIONS_java ?= $(call eol_versions,java) -VARIANT_java ?= noble +VARIANT_java ?= resolute $(foreach javaversion, $(VERSIONS_java), $(eval CONTAINER_java-$(javaversion) = eclipse-temurin:$(javaversion)-jdk-$(VARIANT_java))) CONFIGURE_java ?= java --jars=/usr/share/unit-jsc-common/ INSTALL_java ?= java-shared-install java-install diff --git a/pkg/docker/README.md b/pkg/docker/README.md index a506112dd..2f716223d 100644 --- a/pkg/docker/README.md +++ b/pkg/docker/README.md @@ -187,8 +187,12 @@ A summary (OK / FAILED) is printed at the end. | `wasm` | debian:trixie-slim | | `go-1.25` | golang:1.25 | | `go-1.26` | golang:1.26 | -| `java-17` | eclipse-temurin:17-jdk-noble | -| `java-21` | eclipse-temurin:21-jdk-noble | +| `go-1.27` | golang:1.27 | +| `java-17` | eclipse-temurin:17-jdk-resolute | +| `java-21` | eclipse-temurin:21-jdk-resolute | +| `java-25` | eclipse-temurin:25-jdk-resolute | +| `java-26` | eclipse-temurin:26-jdk-resolute | +| `java-27` | eclipse-temurin:27-jdk-resolute | | `node-20` | node:20 | | `node-22` | node:22 | | `node-24` | node:24 | @@ -196,6 +200,7 @@ A summary (OK / FAILED) is printed at the end. | `perl-5.38` | perl:5.38 | | `perl-5.40` | perl:5.40 | | `perl-5.42` | perl:5.42 | +| `perl-5.44` | perl:5.44 | | `php-8.3` | php:8.3-cli | | `php-8.4` | php:8.4-cli | | `php-8.5` | php:8.5-cli-trixie | diff --git a/pkg/docker/local/Dockerfile.minimal b/pkg/docker/local/Dockerfile.minimal index 5399e34fa..330125a03 100644 --- a/pkg/docker/local/Dockerfile.minimal +++ b/pkg/docker/local/Dockerfile.minimal @@ -77,6 +77,7 @@ COPY --from=build /requirements.apt /requirements.apt RUN set -ex \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get --no-install-recommends --no-install-suggests -y install \ curl $(cat /requirements.apt) \ && rm -rf /var/lib/apt/lists/* \ diff --git a/pkg/docker/local/Dockerfile.php-8.5 b/pkg/docker/local/Dockerfile.php-8.5 index e3643c52f..1774e8e09 100644 --- a/pkg/docker/local/Dockerfile.php-8.5 +++ b/pkg/docker/local/Dockerfile.php-8.5 @@ -77,6 +77,7 @@ COPY --from=build /requirements.apt /requirements.apt RUN set -ex \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get --no-install-recommends --no-install-suggests -y install \ curl $(cat /requirements.apt) \ && rm -rf /var/lib/apt/lists/* \ diff --git a/pkg/docker/local/Dockerfile.wasm b/pkg/docker/local/Dockerfile.wasm index 726db14ab..9af3027d5 100644 --- a/pkg/docker/local/Dockerfile.wasm +++ b/pkg/docker/local/Dockerfile.wasm @@ -81,6 +81,7 @@ COPY --from=build /requirements.apt /requirements.apt RUN set -ex \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get --no-install-recommends --no-install-suggests -y install \ curl $(cat /requirements.apt) \ && rm -rf /var/lib/apt/lists/* \ diff --git a/pkg/docker/template.Dockerfile b/pkg/docker/template.Dockerfile index f80d1300b..44c257ad2 100644 --- a/pkg/docker/template.Dockerfile +++ b/pkg/docker/template.Dockerfile @@ -11,6 +11,7 @@ LABEL org.opencontainers.image.version="@@VERSION@@" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold upgrade \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf libclang-dev cmake \ && export RUST_VERSION=1.95.0 \ diff --git a/pkg/eol.json b/pkg/eol.json index a46e144f3..62b7c3bfe 100644 --- a/pkg/eol.json +++ b/pkg/eol.json @@ -47,13 +47,15 @@ "runtimes": { "go": [ {"version": "1.25", "eol": "2026-08", "supported_until": "2027-08", "note": "EOL"}, - {"version": "1.26", "eol": "2027-02", "supported_until": "2028-02"} + {"version": "1.26", "eol": "2027-02", "supported_until": "2028-02"}, + {"version": "1.27", "eol": "2027-08", "supported_until": "2028-08"} ], "java": [ {"version": "17", "eol": "2027-10", "supported_until": "2028-10", "base": "eclipse-temurin", "lts": true}, {"version": "21", "eol": "2029-12", "supported_until": "2030-12", "base": "eclipse-temurin", "lts": true}, {"version": "25", "eol": "2031-09", "supported_until": "2032-09", "base": "eclipse-temurin", "lts": true}, - {"version": "26", "eol": "2026-09", "supported_until": "2027-09", "base": "eclipse-temurin"} + {"version": "26", "eol": "2026-09", "supported_until": "2027-09", "base": "eclipse-temurin", "note": "EOL"}, + {"version": "27", "eol": "2027-03", "supported_until": "2028-03", "base": "eclipse-temurin"} ], "node": [ {"version": "20", "eol": "2026-04", "supported_until": "2027-04", "note": "EOL"}, @@ -64,7 +66,8 @@ "perl": [ {"version": "5.38", "eol": "2026-07", "supported_until": "2027-07", "note": "EOL"}, {"version": "5.40", "eol": "2027-06", "supported_until": "2028-06"}, - {"version": "5.42", "eol": "2028-07", "supported_until": "2029-07"} + {"version": "5.42", "eol": "2028-07", "supported_until": "2029-07"}, + {"version": "5.44", "eol": "2029-07", "supported_until": "2030-07"} ], "php": [ {"version": "8.3", "eol": "2027-12", "supported_until": "2028-12"}, From 60f5d0c64022d6dd53bf5340fc1b3d6d27c64580 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 17:55:04 +0200 Subject: [PATCH 371/391] rust: declare rust-version per crate, and build the builders with 1.95.0 No crate declared "rust-version", so cargo could not report a toolchain below the floor of a crate. Each crate now declares its floor. Each floor was checked with "cargo check --locked" in the rust: image. The floor passed, and the release before it failed: - src/wasm-wasi-component: 1.95. cranelift 0.135.5 requires 1.95.0. 1.94.1 fails. - tools/unitctl/unitctl and unit-client-rs: 1.95. sysinfo 0.39.6 requires 1.95. 1.94.1 fails. - tools/unitctl/unit-openapi: 1.86. icu_* 2.2.0 require 1.86. 1.85.1 fails. The OpenAPI generator rewrites this Cargo.toml from openapi-templates/Cargo.mustache, so the template gets the line too. - src/otel and test/fake_otlp: 1.88. tonic 0.14.6 requires 1.88. 1.87.0 fails. - pkg/eol and test/fake_upstream: 1.78. Their dependencies declare 1.71 or less, but their Cargo.lock files are version 4. Cargo 1.77.2 cannot parse them. - test/wasm_component/hello_world and spin_loop: 1.78. The test harness builds them for wasm32-wasip1. That target name exists since 1.78. Rust 1.77.2 knows only wasm32-wasi. pkg/docker/Dockerfile.builder-trixie and Dockerfile.builder-php8.5 installed Rust 1.94.1. That is below the 1.95 floor of wasm-wasi-component, which the local wasm variant builds. They now install 1.95.0, as the template and the release Dockerfiles do. The builder image tag carries the Rust version, so the tag moves from rust1.94.1 to rust1.95.0 in build-local.sh, the local/ Dockerfiles and the READMEs. GHCR does not serve this image to an anonymous pull. If the pull fails, build-local.sh builds the image locally, as before. test/run-local.sh mirrors the template, so it moves to 1.95.0 too. pkg/community/CENTOS.ALMALINUX.md now gives 1.88 as the Rust floor for --otel. It said 1.94.1. No dependency changes. Co-Authored-By: Claude Opus 5.5 --- pkg/community/CENTOS.ALMALINUX.md | 4 ++-- pkg/docker/Dockerfile.builder-php8.5 | 2 +- pkg/docker/Dockerfile.builder-trixie | 2 +- pkg/docker/README.md | 4 ++-- pkg/docker/build-local.sh | 4 ++-- pkg/docker/local/Dockerfile.minimal | 4 ++-- pkg/docker/local/Dockerfile.php-8.5 | 4 ++-- pkg/docker/local/Dockerfile.wasm | 4 ++-- pkg/eol/Cargo.toml | 1 + src/otel/Cargo.toml | 1 + src/wasm-wasi-component/Cargo.toml | 1 + test/README.md | 6 +++--- test/fake_otlp/Cargo.toml | 1 + test/fake_upstream/Cargo.toml | 1 + test/run-local.sh | 2 +- test/wasm_component/hello_world/Cargo.toml | 1 + test/wasm_component/spin_loop/Cargo.toml | 1 + tools/unitctl/unit-client-rs/Cargo.toml | 1 + tools/unitctl/unit-openapi/Cargo.toml | 1 + tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache | 1 + tools/unitctl/unitctl/Cargo.toml | 1 + 21 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/community/CENTOS.ALMALINUX.md b/pkg/community/CENTOS.ALMALINUX.md index 6d9ef7ec6..360907e8b 100644 --- a/pkg/community/CENTOS.ALMALINUX.md +++ b/pkg/community/CENTOS.ALMALINUX.md @@ -15,7 +15,7 @@ dnf install -y epel-release dnf install -y gcc make git openssl-devel pcre2-devel \ zlib-devel libzstd-devel brotli-devel -# For OTEL support: Rust >= 1.94.1 (rustup.rs) +# For OTEL support: Rust >= 1.88 (rustup.rs) ``` ### PHP version options @@ -113,7 +113,7 @@ git checkout v1.35.5 # or master / latest tag make -j$(nproc) ``` -> **Note:** `--otel` requires Rust ≥ 1.94.1 (`curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`). +> **Note:** `--otel` requires Rust ≥ 1.88 (`curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`). > Remove `--otel` if you don't need OpenTelemetry support. > > `--user` / `--group` are omitted because the systemd service runs unitd directly. diff --git a/pkg/docker/Dockerfile.builder-php8.5 b/pkg/docker/Dockerfile.builder-php8.5 index 51dc1e892..bbb2d670d 100644 --- a/pkg/docker/Dockerfile.builder-php8.5 +++ b/pkg/docker/Dockerfile.builder-php8.5 @@ -21,7 +21,7 @@ RUN set -ex \ && export CARGO_HOME=/usr/local/cargo \ && export PATH=/usr/local/cargo/bin:$PATH \ && ./rustup-init -y --no-modify-path --profile minimal \ - --default-toolchain 1.94.1 --default-host "${rustArch}" \ + --default-toolchain 1.95.0 --default-host "${rustArch}" \ && rm rustup-init \ && rustc --version \ && cargo --version diff --git a/pkg/docker/Dockerfile.builder-trixie b/pkg/docker/Dockerfile.builder-trixie index 6bfc75343..c2f9940dc 100644 --- a/pkg/docker/Dockerfile.builder-trixie +++ b/pkg/docker/Dockerfile.builder-trixie @@ -21,7 +21,7 @@ RUN set -ex \ && export CARGO_HOME=/usr/local/cargo \ && export PATH=/usr/local/cargo/bin:$PATH \ && ./rustup-init -y --no-modify-path --profile minimal \ - --default-toolchain 1.94.1 --default-host "${rustArch}" \ + --default-toolchain 1.95.0 --default-host "${rustArch}" \ && rm rustup-init \ && rustc --version \ && cargo --version diff --git a/pkg/docker/README.md b/pkg/docker/README.md index 2f716223d..372aadcfa 100644 --- a/pkg/docker/README.md +++ b/pkg/docker/README.md @@ -166,8 +166,8 @@ Builder images (`Dockerfile.builder-*`) are pulled from GHCR automatically, or built locally on first use if unavailable: ``` -Dockerfile.builder-trixie → ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 -Dockerfile.builder-php8.5 → ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.94.1 +Dockerfile.builder-trixie → ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 +Dockerfile.builder-php8.5 → ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.95.0 ``` The `local/` subdirectory contains the corresponding multi-stage Dockerfiles used diff --git a/pkg/docker/build-local.sh b/pkg/docker/build-local.sh index bb82c013d..21543a6ba 100755 --- a/pkg/docker/build-local.sh +++ b/pkg/docker/build-local.sh @@ -175,9 +175,9 @@ ensure_builder() { # silently fall back to a local build. local IMG BDF case "$VARIANT" in - minimal|wasm) IMG="ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1" + minimal|wasm) IMG="ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0" BDF="${SCRIPT_DIR}/Dockerfile.builder-trixie" ;; - php-8.5) IMG="ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.94.1" + php-8.5) IMG="ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.95.0" BDF="${SCRIPT_DIR}/Dockerfile.builder-php8.5" ;; *) return 0 ;; # no builder for this variant esac diff --git a/pkg/docker/local/Dockerfile.minimal b/pkg/docker/local/Dockerfile.minimal index 330125a03..1833623d9 100644 --- a/pkg/docker/local/Dockerfile.minimal +++ b/pkg/docker/local/Dockerfile.minimal @@ -1,11 +1,11 @@ # Fast local build — uses pre-built builder image (no apt/Rust download). # Build builder first: -# docker build -t ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 \ +# docker build -t ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 \ # -f ../Dockerfile.builder-trixie .. # Then build this variant: # ../build-local.sh -b minimal -FROM ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 AS build +FROM ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 AS build RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ diff --git a/pkg/docker/local/Dockerfile.php-8.5 b/pkg/docker/local/Dockerfile.php-8.5 index 1774e8e09..6a73c9c3d 100644 --- a/pkg/docker/local/Dockerfile.php-8.5 +++ b/pkg/docker/local/Dockerfile.php-8.5 @@ -1,11 +1,11 @@ # Fast local build — uses pre-built builder image (no apt/Rust download). # Build builder first: -# docker build -t ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.94.1 \ +# docker build -t ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.95.0 \ # -f ../Dockerfile.builder-php8.5 .. # Then build this variant: # ../build-local.sh -b php-8.5 -FROM ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.94.1 AS build +FROM ghcr.io/freeunitorg/freeunit-builder:php8.5-rust1.95.0 AS build RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ diff --git a/pkg/docker/local/Dockerfile.wasm b/pkg/docker/local/Dockerfile.wasm index 9af3027d5..9a3767bf3 100644 --- a/pkg/docker/local/Dockerfile.wasm +++ b/pkg/docker/local/Dockerfile.wasm @@ -1,8 +1,8 @@ # Fast local build — uses pre-built builder image (no apt/Rust download). -# Builder: ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 +# Builder: ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 # (same as minimal — wasmtime Rust build is included in that builder) -FROM ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 AS build +FROM ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 AS build RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ diff --git a/pkg/eol/Cargo.toml b/pkg/eol/Cargo.toml index c72f878b7..68b514eec 100644 --- a/pkg/eol/Cargo.toml +++ b/pkg/eol/Cargo.toml @@ -2,6 +2,7 @@ name = "unit-eol-check" version = "1.0.0" edition = "2021" +rust-version = "1.78" [dependencies] serde_json = "1" diff --git a/src/otel/Cargo.toml b/src/otel/Cargo.toml index a8c2ab834..86ab6afd1 100644 --- a/src/otel/Cargo.toml +++ b/src/otel/Cargo.toml @@ -2,6 +2,7 @@ name = "otel" version = "0.1.0" edition = "2021" +rust-version = "1.88" [lib] crate-type = ["staticlib"] diff --git a/src/wasm-wasi-component/Cargo.toml b/src/wasm-wasi-component/Cargo.toml index f542d0fe6..f591bc11f 100644 --- a/src/wasm-wasi-component/Cargo.toml +++ b/src/wasm-wasi-component/Cargo.toml @@ -2,6 +2,7 @@ name = "wasm-wasi-component" version = "0.1.0" edition = "2021" +rust-version = "1.95" publish = false [lib] diff --git a/test/README.md b/test/README.md index 9d98af2cc..7460c88f9 100644 --- a/test/README.md +++ b/test/README.md @@ -53,7 +53,7 @@ build tools. ``` The script (`test/run-local.sh`) builds a `freeunit-test:local` image -mirroring `pkg/docker/template.Dockerfile` (Debian trixie, Rust 1.94.1, +mirroring `pkg/docker/template.Dockerfile` (Debian trixie, Rust 1.95.0, njs 0.9.6, system libssl-dev). Source code is mounted via Docker volume, so changes on the host are immediately reflected. The container then builds FreeUnit with `--tests --openssl --njs --zlib --zstd --brotli --otel`, @@ -70,12 +70,12 @@ docker rmi freeunit-test:local `run-local.sh` builds a full test image from scratch (downloads Rust, Go, njs) — slow for tight iteration. For prototyping a **proxy / TLS** test (no language runtime needed), reuse the pre-built builder image -`ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1` (Rust + all C build +`ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0` (Rust + all C build deps already baked in) and just mount the working tree. Build + run is ~30 s: ```bash docker run --rm --privileged -v "$(pwd):/unit" -w /unit \ - ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.94.1 bash -c ' + ghcr.io/freeunitorg/freeunit-builder:trixie-rust1.95.0 bash -c ' apt-get update -qq && apt-get install -y -qq python3-pytest python3-openssl ./configure --openssl --tests make -j"$(nproc)" unitd diff --git a/test/fake_otlp/Cargo.toml b/test/fake_otlp/Cargo.toml index ed7b8dabb..5c8549c8a 100644 --- a/test/fake_otlp/Cargo.toml +++ b/test/fake_otlp/Cargo.toml @@ -2,6 +2,7 @@ name = "fake_otlp" version = "0.1.0" edition = "2021" +rust-version = "1.88" [[bin]] name = "fake_otlp" diff --git a/test/fake_upstream/Cargo.toml b/test/fake_upstream/Cargo.toml index e7c04114d..d5e5cb509 100644 --- a/test/fake_upstream/Cargo.toml +++ b/test/fake_upstream/Cargo.toml @@ -2,6 +2,7 @@ name = "fake_upstream" version = "0.1.0" edition = "2021" +rust-version = "1.78" [[bin]] name = "fake_upstream" diff --git a/test/run-local.sh b/test/run-local.sh index 1d9975cac..919937d18 100755 --- a/test/run-local.sh +++ b/test/run-local.sh @@ -180,7 +180,7 @@ RUN set -ex \ ca-certificates git build-essential libssl-dev openssl libpcre2-dev \ zlib1g-dev libzstd-dev libbrotli-dev curl wget pkg-config pkgconf \ libclang-dev cmake python3-pytest python3-openssl sudo procps \ - && export RUST_VERSION=1.94.1 \ + && export RUST_VERSION=1.95.0 \ && export RUSTUP_HOME=/usr/src/unit/rustup \ && export CARGO_HOME=/usr/src/unit/cargo \ && export PATH=/usr/src/unit/cargo/bin:$PATH \ diff --git a/test/wasm_component/hello_world/Cargo.toml b/test/wasm_component/hello_world/Cargo.toml index a87fbeb5a..46760c8bf 100644 --- a/test/wasm_component/hello_world/Cargo.toml +++ b/test/wasm_component/hello_world/Cargo.toml @@ -2,6 +2,7 @@ name = "test-wasi-component" version = "0.1.0" edition = "2021" +rust-version = "1.78" [dependencies] bitflags = "2.4.2" diff --git a/test/wasm_component/spin_loop/Cargo.toml b/test/wasm_component/spin_loop/Cargo.toml index a87fbeb5a..46760c8bf 100644 --- a/test/wasm_component/spin_loop/Cargo.toml +++ b/test/wasm_component/spin_loop/Cargo.toml @@ -2,6 +2,7 @@ name = "test-wasi-component" version = "0.1.0" edition = "2021" +rust-version = "1.78" [dependencies] bitflags = "2.4.2" diff --git a/tools/unitctl/unit-client-rs/Cargo.toml b/tools/unitctl/unit-client-rs/Cargo.toml index bf34270ff..ef7755225 100644 --- a/tools/unitctl/unit-client-rs/Cargo.toml +++ b/tools/unitctl/unit-client-rs/Cargo.toml @@ -3,6 +3,7 @@ name = "unit-client-rs" version = "1.36.1" authors = ["Elijah Zupancic"] edition = "2021" +rust-version = "1.95" license = "Apache-2.0" [lib] diff --git a/tools/unitctl/unit-openapi/Cargo.toml b/tools/unitctl/unit-openapi/Cargo.toml index 4e4f14959..f3626d2df 100644 --- a/tools/unitctl/unit-openapi/Cargo.toml +++ b/tools/unitctl/unit-openapi/Cargo.toml @@ -5,6 +5,7 @@ authors = ["team@freeunit.org"] description = "FreeUnit (ex NGINX Unit) is a lightweight and versatile application runtime that provides the essential components for your web application as a single open-source server: running application code, serving static assets, handling TLS and request routing. **Important**: Unit's API is designed to expose any part of its configuration as an addressable endpoint. Suppose a JSON object is stored at `/config/listeners/`: ```json { \"*:8080\": { \"pass\": \"applications/wp_emea_dev\" } } ``` Here, `/config/listeners/_*:8080` and `/config/listeners/_*:8080/pass` are also endpoints. Generally, object options are addressable by their names, array items—by their indexes (`/array/0/`). **Note**: By default, Unit is configured through a UNIX domain socket. To use this specification with OpenAPI tools interactively, start Unit with a TCP port as the control socket." license = "Apache 2.0" edition = "2018" +rust-version = "1.86" [dependencies] serde = { version = "1.0", features = ["derive"] } diff --git a/tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache b/tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache index c95a9625e..864f60084 100644 --- a/tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache +++ b/tools/unitctl/unit-openapi/openapi-templates/Cargo.mustache @@ -18,6 +18,7 @@ license = "{{.}}" license = "Unlicense" {{/licenseInfo}} edition = "2018" +rust-version = "1.86" {{#publishRustRegistry}} publish = ["{{.}}"] {{/publishRustRegistry}} diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index 3c22fa1a9..a70d47d42 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -4,6 +4,7 @@ description = "CLI interface to the FreeUnit Control API" version = "1.36.1" authors = ["Elijah Zupancic"] edition = "2021" +rust-version = "1.95" license = "Apache-2.0" [[bin]] From fe2a09324a73c43c338650dc314f4e091491daf4 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 18:41:07 +0200 Subject: [PATCH 372/391] ci: build every crate with its declared rust-version The pull request jobs build the Rust crates with stable only. build-deb.yml pins Rust 1.88.0 for src/otel, but it does not run on pull requests. A Dependabot update that raises the Rust floor of a dependency could pass its pull request and break the deb build. Codex found this on https://github.com/freeunitorg/freeunit/pull/565. The new workflow .github/workflows/rust-version.yml has one leg for each directory with its own Cargo.lock. These are the same eight directories as the cargo entries in .github/dependabot.yml. Each leg runs .github/scripts/check-rust-version.py. For every package, the script reads "rust-version" from Cargo.toml, or from the workspace when the package inherits it. It installs that toolchain with rustup and runs "cargo + check --locked" on the package. A package without "rust-version" fails, so a floor cannot disappear silently. Some legs need more setup: - tools/unitctl generates the OpenAPI client first, and then restores the tracked unit-openapi/Cargo.toml that the generator rewrites. - src/wasm-wasi-component installs libclang-dev and builds build/src/nxt_unit.o for its build script. - The two test/wasm_component crates are checked for wasm32-wasip1. The workflow runs on pull requests and on pushes to master that change a Cargo.toml, a Cargo.lock, a rust-toolchain.toml, the OpenAPI template or spec, or the workflow and its script. It uses only actions/checkout, pinned by SHA, and the rustup that the runner has. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check-rust-version.py | 66 ++++++++++++++++++++++ .github/workflows/rust-version.yml | 81 +++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100755 .github/scripts/check-rust-version.py create mode 100644 .github/workflows/rust-version.yml diff --git a/.github/scripts/check-rust-version.py b/.github/scripts/check-rust-version.py new file mode 100755 index 000000000..e6c228096 --- /dev/null +++ b/.github/scripts/check-rust-version.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# +# Check every package of a Cargo project with the Rust release that its +# "rust-version" names, so that a dependency update which raises the floor +# fails here instead of in a build that pins an older toolchain. +# +# Usage: check-rust-version.py [] +# +# A package without "rust-version" is an error. + +import os +import subprocess +import sys +import tomllib + + +def load(path): + with open(path, 'rb') as f: + return tomllib.load(f) + + +def run(cmd): + print('+ ' + ' '.join(cmd), flush=True) + return subprocess.run(cmd).returncode + + +def main(): + root = sys.argv[1] + target = ['--target', sys.argv[2]] if len(sys.argv) > 2 else [] + manifest = os.path.join(root, 'Cargo.toml') + top = load(manifest) + workspace = top.get('workspace', {}) + + dirs = [os.path.join(root, m) for m in workspace.get('members', [])] + if 'package' in top: + dirs.insert(0, root) + + failed = False + + for d in dirs: + path = os.path.join(d, 'Cargo.toml') + package = load(path)['package'] + version = package.get('rust-version') + + if isinstance(version, dict) and version.get('workspace'): + version = workspace.get('package', {}).get('rust-version') + + if not isinstance(version, str): + print(f'::error file={path}::{package["name"]} declares no ' + 'rust-version') + failed = True + continue + + if (run(['rustup', 'toolchain', 'install', version, + '--profile', 'minimal', *target]) != 0 + or run(['cargo', f'+{version}', 'check', '--locked', + '--manifest-path', manifest, '-p', package['name'], + *target]) != 0): + print(f'::error file={path}::{package["name"]} does not build ' + f'with Rust {version}, its rust-version') + failed = True + + sys.exit(1 if failed else 0) + + +main() diff --git a/.github/workflows/rust-version.yml b/.github/workflows/rust-version.yml new file mode 100644 index 000000000..3592e44af --- /dev/null +++ b/.github/workflows/rust-version.yml @@ -0,0 +1,81 @@ +name: "Rust (rust-version)" + +# Builds every crate with the Rust release its "rust-version" declares. +# The other pull request jobs build the crates with stable Rust only. +# build-deb.yml pins Rust 1.88.0, but it does not run on pull requests. So +# a dependency update that raises a crate's floor could pass its pull +# request and then break the deb build. There is one leg per directory +# with its own Cargo.lock, the same directories as the cargo entries in +# .github/dependabot.yml. A package without "rust-version" fails its leg. + +on: + pull_request: + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + - '**/rust-toolchain.toml' + - 'tools/unitctl/unit-openapi/openapi-templates/**' + - 'docs/unit-openapi.yaml' + - '.github/scripts/check-rust-version.py' + - '.github/workflows/rust-version.yml' + push: + branches: master + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + - '**/rust-toolchain.toml' + - 'tools/unitctl/unit-openapi/openapi-templates/**' + - 'docs/unit-openapi.yaml' + - '.github/scripts/check-rust-version.py' + - '.github/workflows/rust-version.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - dir: tools/unitctl + setup: openapi + - dir: src/otel + - dir: src/wasm-wasi-component + setup: libunit + - dir: pkg/eol + - dir: test/fake_otlp + - dir: test/fake_upstream + - dir: test/wasm_component/hello_world + target: wasm32-wasip1 + - dir: test/wasm_component/spin_loop + target: wasm32-wasip1 + name: ${{ matrix.dir }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The OpenAPI client is generated, not tracked. The generator also + # rewrites unit-openapi/Cargo.toml, so the tracked file is restored. + - name: Generate the OpenAPI client + if: matrix.setup == 'openapi' + run: | + make -C tools/unitctl CARGO=true openapi-generate + git checkout -- tools/unitctl/unit-openapi/Cargo.toml + + # build.rs runs bindgen on the Unit headers and links nxt_unit.o. + - name: Build nxt_unit.o + if: matrix.setup == 'libunit' + run: | + sudo apt-get update + sudo apt-get install -y libclang-dev + ./configure --no-regex + make -j"$(nproc)" build/src/nxt_unit.o + + - name: cargo check with rust-version + run: python3 .github/scripts/check-rust-version.py ${{ matrix.dir }} ${{ matrix.target }} From f031a0b2789ee5a22d2a18091274625681199ca2 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 19:14:27 +0200 Subject: [PATCH 373/391] ci: fail the wasm-wasi-component leg when its tests all skip test/unit/applications/lang/wasm_component.py returns None when "cargo component build" fails. test/unit/check/cargo_component.py then reports the cargo_component feature as missing, and test/test_wasm-wasi-component.py skips as a whole. The leg stayed green while it tested nothing. So a dependency update that passes "cargo check" in rust-version.yml but breaks the component build could pass its pull request. The step "Fail if the wasm smoke test skipped" already covered the wasm leg, and its pytest output is already saved for both legs. The step now runs for the wasm-wasi-component leg too. It requires a PASSED line from test_.py: test_wasm.py for wasm, and test_wasm-wasi-component.py for wasm-wasi-component. The comment on the build list said 23 legs. With Go 1.27 and Java 27 the matrix has 25. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b71f37440..f355784f4 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v7 - name: Derive build list from pkg/eol.json id: list - # Slow legs come first in the list. All 23 legs are created at + # Slow legs come first in the list. All 25 legs are created at # once. The hosted runner pool is smaller than the matrix, so some # legs wait for a runner. GitHub does not promise to start legs in # matrix order. Two runs (36318226905 and 36322425869) showed no @@ -924,12 +924,15 @@ jobs: # check_prerequisites() skips the whole file when the clang/wasi-sysroot # fixture cannot be built, so an incomplete toolchain would leave this # leg green while it covers nothing. At least one smoke test must have - # actually passed. + # actually passed. The wasm-wasi-component leg has the same trap: a + # failed "cargo component build" turns into a missing cargo_component + # feature, and test_wasm-wasi-component.py skips as a whole. - name: Fail if the wasm smoke test skipped - if: ${{ !cancelled() && steps.metadata.outputs.module == 'wasm' }} + if: ${{ !cancelled() && (steps.metadata.outputs.module == 'wasm' || steps.metadata.outputs.module == 'wasm-wasi-component') }} run: | - grep -qE 'test_wasm\.py::.*PASSED' "$RUNNER_TEMP/pytest.out" || { - echo "no test_wasm.py test passed; the guest toolchain skipped it:" >&2 + file="test_${{ steps.metadata.outputs.module }}.py" + grep -qE "${file//./\\.}::.*PASSED" "$RUNNER_TEMP/pytest.out" || { + echo "no $file test passed; the guest toolchain skipped it:" >&2 grep -E 'SKIPPED|feature' "$RUNNER_TEMP/pytest.out" >&2 || true exit 1 } From 61b297acd91aa6f04ecaf5ce9908751469a3af53 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 12:23:59 +0000 Subject: [PATCH 374/391] release: prepare 1.37.0 1.36.2 was never tagged, and what master carries since 1.36.1 includes two features (byte ranges and conditional requests for static files), so the next release is 1.37.0 rather than a patch. The changelog block headed 1.36.2 becomes the 1.37.0 block; nothing in it is dropped. version NXT_VERSION 1.37.0, NXT_VERNUM 13700 docs/changes.xml both 1.36.2 blocks retitled, date set CHANGES header retitled by hand (see below) docs/unit-openapi.yaml title tools/unitctl crate versions, Cargo.lock, openapi-config, README, and a 1.37.0 CHANGELOG section for the unitctl changes merged since 1.36.1 pkg/docker "make dockerfiles"; the diff is the version only CHANGES, changes.xml two entries for access-log changes that had none: 279e1bb5 (object-format values escaped) and 31fcf3e4 (invalid UTF-8 becomes U+FFFD) CHANGES is edited by hand, not regenerated: "make -C docs changes" titles the release "Changes with Unit" and wraps every entry differently from the file the changelog commits have been maintaining. The date, 2026-10-04, is the tag date. This commit is rebased onto master at the freeze and has to be the last one merged before the tag, since every changelog commit in the merge train touches the same block. --- CHANGES | 14 +++++++++++- docs/changes.xml | 28 +++++++++++++++++++----- docs/unit-openapi.yaml | 2 +- pkg/docker/Dockerfile.go-1.25 | 4 ++-- pkg/docker/Dockerfile.go-1.26 | 4 ++-- pkg/docker/Dockerfile.java-17 | 4 ++-- pkg/docker/Dockerfile.java-21 | 4 ++-- pkg/docker/Dockerfile.java-25 | 4 ++-- pkg/docker/Dockerfile.java-26 | 4 ++-- pkg/docker/Dockerfile.minimal | 4 ++-- pkg/docker/Dockerfile.node-20 | 4 ++-- pkg/docker/Dockerfile.node-22 | 4 ++-- pkg/docker/Dockerfile.node-24 | 4 ++-- pkg/docker/Dockerfile.node-26 | 4 ++-- pkg/docker/Dockerfile.perl-5.38 | 4 ++-- pkg/docker/Dockerfile.perl-5.40 | 4 ++-- pkg/docker/Dockerfile.perl-5.42 | 4 ++-- pkg/docker/Dockerfile.php-8.3 | 4 ++-- pkg/docker/Dockerfile.php-8.4 | 4 ++-- pkg/docker/Dockerfile.php-8.5 | 4 ++-- pkg/docker/Dockerfile.python-3.12 | 4 ++-- pkg/docker/Dockerfile.python-3.12-slim | 4 ++-- pkg/docker/Dockerfile.python-3.13 | 4 ++-- pkg/docker/Dockerfile.python-3.13-slim | 4 ++-- pkg/docker/Dockerfile.python-3.14 | 4 ++-- pkg/docker/Dockerfile.python-3.14-slim | 4 ++-- pkg/docker/Dockerfile.ruby-3.3 | 4 ++-- pkg/docker/Dockerfile.ruby-3.4 | 4 ++-- pkg/docker/Dockerfile.ruby-4.0 | 4 ++-- pkg/docker/Dockerfile.wasm | 4 ++-- tools/unitctl/CHANGELOG.md | 29 ++++++++++++++++++++++++- tools/unitctl/Cargo.lock | 6 ++--- tools/unitctl/openapi-config.json | 2 +- tools/unitctl/unit-client-rs/Cargo.toml | 2 +- tools/unitctl/unit-openapi/Cargo.toml | 2 +- tools/unitctl/unit-openapi/README.md | 2 +- tools/unitctl/unitctl/Cargo.toml | 2 +- version | 4 ++-- 38 files changed, 129 insertions(+), 72 deletions(-) diff --git a/CHANGES b/CHANGES index 4b503f14d..be7818c9b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -Changes with FreeUnit 1.36.2 30 Aug 2026 +Changes with FreeUnit 1.37.0 04 Oct 2026 *) Bugfix: a chunked request body that grew over "max_body_size" after the read that carried the request header got no response. @@ -471,6 +471,18 @@ Changes with FreeUnit 1.36.2 30 Aug 2026 but it is not a way to keep bytes either: it replaces anything that is not valid UTF-8 with U+FFFD before it serializes. + *) Security: when "access_log" "format" was a JSON object without an + njs expression, the router wrote each variable's value into the + record without escaping it. A request header with a quote could + close a member and add members to the record. Every value is now + escaped. A variable in a member name is no longer expanded; the + name is written as configured. + + *) Change: an object-format access log replaces each byte that begins + no valid UTF-8 sequence with U+FFFD. Request headers may carry such + bytes, and the record was then not valid JSON. The string form + keeps them, written as \xHH. + *) Change: upgrade contrib njs to 1.0.1. *) Bugfix: a port that was closed and freed could leave a pending epoll diff --git a/docs/changes.xml b/docs/changes.xml index 8dfdaaf84..7a5494088 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -6,8 +6,8 @@ @@ -659,6 +659,24 @@ replaces anything that is not valid UTF-8 with U+FFFD before it serializes. + + +when "access_log" "format" was a JSON object without an njs expression, the +router wrote each variable's value into the record without escaping it. A +request header with a quote could close a member and add members to the +record. Every value is now escaped. A variable in a member name is no longer +expanded; the name is written as configured. + + + + + +an object-format access log replaces each byte that begins no valid UTF-8 +sequence with U+FFFD. Request headers may carry such bytes, and the record was +then not valid JSON. The string form keeps them, written as \xHH. + + + a port message that was queued because the peer's socket was not writable @@ -1353,13 +1371,13 @@ were when that image was published. unit-jsc-common unit-jsc11 unit-jsc17 unit-jsc18 unit-jsc19 unit-jsc20 unit-jsc21 unit-wasm" - ver="1.36.2" rev="1" - date="2026-08-30" time="00:00:00 +0000" + ver="1.37.0" rev="1" + date="2026-10-04" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> -FreeUnit updated to 1.36.2. +FreeUnit updated to 1.37.0. diff --git a/docs/unit-openapi.yaml b/docs/unit-openapi.yaml index 14d0b6276..b9cb6ab95 100644 --- a/docs/unit-openapi.yaml +++ b/docs/unit-openapi.yaml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - title: "FreeUnit 1.36.1 (ex NGINX Unit)" + title: "FreeUnit 1.37.0 (ex NGINX Unit)" description: "FreeUnit (ex NGINX Unit) is a lightweight and versatile application runtime that provides the essential components for your web application as a single open-source server: running application code, serving static assets, diff --git a/pkg/docker/Dockerfile.go-1.25 b/pkg/docker/Dockerfile.go-1.25 index 95862cb6b..1bd99149a 100644 --- a/pkg/docker/Dockerfile.go-1.25 +++ b/pkg/docker/Dockerfile.go-1.25 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.go-1.26 b/pkg/docker/Dockerfile.go-1.26 index 57979bd29..6478d6aa2 100644 --- a/pkg/docker/Dockerfile.go-1.26 +++ b/pkg/docker/Dockerfile.go-1.26 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-17 b/pkg/docker/Dockerfile.java-17 index fae252503..520771b66 100644 --- a/pkg/docker/Dockerfile.java-17 +++ b/pkg/docker/Dockerfile.java-17 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-21 b/pkg/docker/Dockerfile.java-21 index 7a5eadc31..ecb9d13b7 100644 --- a/pkg/docker/Dockerfile.java-21 +++ b/pkg/docker/Dockerfile.java-21 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-25 b/pkg/docker/Dockerfile.java-25 index ea196a7a6..faf03ca7e 100644 --- a/pkg/docker/Dockerfile.java-25 +++ b/pkg/docker/Dockerfile.java-25 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.java-26 b/pkg/docker/Dockerfile.java-26 index c9dbdef68..8ef41b59f 100644 --- a/pkg/docker/Dockerfile.java-26 +++ b/pkg/docker/Dockerfile.java-26 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index 4c7a3fb72..fee881187 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-20 b/pkg/docker/Dockerfile.node-20 index ef505a8e7..4c03fd673 100644 --- a/pkg/docker/Dockerfile.node-20 +++ b/pkg/docker/Dockerfile.node-20 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-22 b/pkg/docker/Dockerfile.node-22 index 87cc48677..691352bab 100644 --- a/pkg/docker/Dockerfile.node-22 +++ b/pkg/docker/Dockerfile.node-22 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-24 b/pkg/docker/Dockerfile.node-24 index 56fce9ae4..7f613141b 100644 --- a/pkg/docker/Dockerfile.node-24 +++ b/pkg/docker/Dockerfile.node-24 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.node-26 b/pkg/docker/Dockerfile.node-26 index c38481b4c..c0755af5b 100644 --- a/pkg/docker/Dockerfile.node-26 +++ b/pkg/docker/Dockerfile.node-26 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.38 b/pkg/docker/Dockerfile.perl-5.38 index 3a11ca109..75982c269 100644 --- a/pkg/docker/Dockerfile.perl-5.38 +++ b/pkg/docker/Dockerfile.perl-5.38 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.40 b/pkg/docker/Dockerfile.perl-5.40 index ed4a20cc7..9dbaab13f 100644 --- a/pkg/docker/Dockerfile.perl-5.40 +++ b/pkg/docker/Dockerfile.perl-5.40 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.perl-5.42 b/pkg/docker/Dockerfile.perl-5.42 index 66222a286..c4ae23512 100644 --- a/pkg/docker/Dockerfile.perl-5.42 +++ b/pkg/docker/Dockerfile.perl-5.42 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.php-8.3 b/pkg/docker/Dockerfile.php-8.3 index f5070d033..e3b049ef6 100644 --- a/pkg/docker/Dockerfile.php-8.3 +++ b/pkg/docker/Dockerfile.php-8.3 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.php-8.4 b/pkg/docker/Dockerfile.php-8.4 index 677965517..779ef2bef 100644 --- a/pkg/docker/Dockerfile.php-8.4 +++ b/pkg/docker/Dockerfile.php-8.4 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.php-8.5 b/pkg/docker/Dockerfile.php-8.5 index 441f54b13..d9bc4d514 100644 --- a/pkg/docker/Dockerfile.php-8.5 +++ b/pkg/docker/Dockerfile.php-8.5 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.12 b/pkg/docker/Dockerfile.python-3.12 index 7c3275b1c..257cd6124 100644 --- a/pkg/docker/Dockerfile.python-3.12 +++ b/pkg/docker/Dockerfile.python-3.12 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.12-slim b/pkg/docker/Dockerfile.python-3.12-slim index d4cb2e4d7..bb237321a 100644 --- a/pkg/docker/Dockerfile.python-3.12-slim +++ b/pkg/docker/Dockerfile.python-3.12-slim @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.13 b/pkg/docker/Dockerfile.python-3.13 index 99122ead0..1087c0b86 100644 --- a/pkg/docker/Dockerfile.python-3.13 +++ b/pkg/docker/Dockerfile.python-3.13 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.13-slim b/pkg/docker/Dockerfile.python-3.13-slim index 7a883d2af..59ac35ab5 100644 --- a/pkg/docker/Dockerfile.python-3.13-slim +++ b/pkg/docker/Dockerfile.python-3.13-slim @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.14 b/pkg/docker/Dockerfile.python-3.14 index 78daa290b..1652a1a6b 100644 --- a/pkg/docker/Dockerfile.python-3.14 +++ b/pkg/docker/Dockerfile.python-3.14 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.python-3.14-slim b/pkg/docker/Dockerfile.python-3.14-slim index f829addea..37daae820 100644 --- a/pkg/docker/Dockerfile.python-3.14-slim +++ b/pkg/docker/Dockerfile.python-3.14-slim @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-3.3 b/pkg/docker/Dockerfile.ruby-3.3 index 991716652..bf4d0e64c 100644 --- a/pkg/docker/Dockerfile.ruby-3.3 +++ b/pkg/docker/Dockerfile.ruby-3.3 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-3.4 b/pkg/docker/Dockerfile.ruby-3.4 index 7f7e82157..922815cf6 100644 --- a/pkg/docker/Dockerfile.ruby-3.4 +++ b/pkg/docker/Dockerfile.ruby-3.4 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.ruby-4.0 b/pkg/docker/Dockerfile.ruby-4.0 index f22d77dd3..fe11f9723 100644 --- a/pkg/docker/Dockerfile.ruby-4.0 +++ b/pkg/docker/Dockerfile.ruby-4.0 @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/pkg/docker/Dockerfile.wasm b/pkg/docker/Dockerfile.wasm index a867fa52b..096e1020d 100644 --- a/pkg/docker/Dockerfile.wasm +++ b/pkg/docker/Dockerfile.wasm @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.url="https://freeunit.org" LABEL org.opencontainers.image.source="https://github.com/freeunitorg/freeunit" LABEL org.opencontainers.image.documentation="https://github.com/freeunitorg/freeunit/blob/master/README.md" LABEL org.opencontainers.image.vendor="FreeUnit Community " -LABEL org.opencontainers.image.version="1.36.1" +LABEL org.opencontainers.image.version="1.37.0" RUN set -ex \ && savedAptMark="$(apt-mark showmanual)" \ @@ -36,7 +36,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ - && git clone --depth 1 -b 1.36.1 https://github.com/freeunitorg/freeunit unit \ + && git clone --depth 1 -b 1.37.0 https://github.com/freeunitorg/freeunit unit \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ diff --git a/tools/unitctl/CHANGELOG.md b/tools/unitctl/CHANGELOG.md index 67e1833d1..60cbee592 100644 --- a/tools/unitctl/CHANGELOG.md +++ b/tools/unitctl/CHANGELOG.md @@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.37.0] - 2026-10-04 + +### Removed +- JSON5 input. A `.json5` configuration file is refused by name, as hjson and YAML already + were, and the message names a converter. `edit` reads its temporary file as JSON, so a + comment written in the editor is now a syntax error. + +### Fixed +- `execute -m PUT -p /js_modules/.js -f .js` sends the module + (`Content-Type: application/javascript`, body verbatim) instead of panicking with + "Unknown input file type". A file whose type cannot be placed is refused with an error in + both `execute` and `import` rather than aborting. +- A JSON configuration file is sent as it was written. It was parsed into a map and + re-serialized, which silently merged duplicate member names and re-spelled numbers; the + server now sees the file and reports a duplicate member with its line and column. +- `save` no longer tells the operator to restore a configuration that holds non-UTF-8 bytes + with curl; the control API refuses such a configuration whichever client sends it, so the + message now names the value that has to be corrected. + +### Security +- rustls 0.23.42 → 0.23.45 (RUSTSEC-2026-0285, GHSA-2mjx-qc3c-rqvc), with aws-lc-rs 1.18.1, + aws-lc-sys 0.45.0 and rustls-webpki 0.103.15. + +### Changed +- Version bump to track the FreeUnit 1.37.0 release. + ## [1.36.1] - 2026-08-28 ### Fixed @@ -68,7 +94,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Previous releases See earlier versions for additional change history. -[Unreleased]: https://github.com/freeunitorg/freeunit/compare/unitctl/1.36.1...HEAD +[Unreleased]: https://github.com/freeunitorg/freeunit/compare/unitctl/1.37.0...HEAD +[1.37.0]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.37.0 [1.36.1]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.36.1 [1.36.0]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.36.0 [1.35.5]: https://github.com/freeunitorg/freeunit/releases/tag/unitctl/1.35.5 diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 2244999cf..63953a0a3 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -1435,7 +1435,7 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unit-client-rs" -version = "1.36.1" +version = "1.37.0" dependencies = [ "bollard", "bytes", @@ -1461,7 +1461,7 @@ dependencies = [ [[package]] name = "unit-openapi" -version = "1.36.1" +version = "1.37.0" dependencies = [ "base64", "bytes", @@ -1477,7 +1477,7 @@ dependencies = [ [[package]] name = "unitctl" -version = "1.36.1" +version = "1.37.0" dependencies = [ "bytes", "clap", diff --git a/tools/unitctl/openapi-config.json b/tools/unitctl/openapi-config.json index 535313dd7..723adbb67 100644 --- a/tools/unitctl/openapi-config.json +++ b/tools/unitctl/openapi-config.json @@ -1,6 +1,6 @@ { "packageName": "unit-openapi", - "packageVersion": "1.36.1", + "packageVersion": "1.37.0", "library": "hyper", "preferUnsignedInt": true } diff --git a/tools/unitctl/unit-client-rs/Cargo.toml b/tools/unitctl/unit-client-rs/Cargo.toml index ef7755225..b4e0ae788 100644 --- a/tools/unitctl/unit-client-rs/Cargo.toml +++ b/tools/unitctl/unit-client-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "unit-client-rs" -version = "1.36.1" +version = "1.37.0" authors = ["Elijah Zupancic"] edition = "2021" rust-version = "1.95" diff --git a/tools/unitctl/unit-openapi/Cargo.toml b/tools/unitctl/unit-openapi/Cargo.toml index f3626d2df..d371a86f3 100644 --- a/tools/unitctl/unit-openapi/Cargo.toml +++ b/tools/unitctl/unit-openapi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "unit-openapi" -version = "1.36.1" +version = "1.37.0" authors = ["team@freeunit.org"] description = "FreeUnit (ex NGINX Unit) is a lightweight and versatile application runtime that provides the essential components for your web application as a single open-source server: running application code, serving static assets, handling TLS and request routing. **Important**: Unit's API is designed to expose any part of its configuration as an addressable endpoint. Suppose a JSON object is stored at `/config/listeners/`: ```json { \"*:8080\": { \"pass\": \"applications/wp_emea_dev\" } } ``` Here, `/config/listeners/_*:8080` and `/config/listeners/_*:8080/pass` are also endpoints. Generally, object options are addressable by their names, array items—by their indexes (`/array/0/`). **Note**: By default, Unit is configured through a UNIX domain socket. To use this specification with OpenAPI tools interactively, start Unit with a TCP port as the control socket." license = "Apache 2.0" diff --git a/tools/unitctl/unit-openapi/README.md b/tools/unitctl/unit-openapi/README.md index dfddcbd67..ba1f1a1f0 100644 --- a/tools/unitctl/unit-openapi/README.md +++ b/tools/unitctl/unit-openapi/README.md @@ -21,7 +21,7 @@ For more information, please visit [https://github.com/freeunitorg/freeunit](htt This API client was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the [openapi-spec](https://openapis.org) from a remote server, you can easily generate an API client. - API version: 0.2.0 -- Package version: 1.36.1 +- Package version: 1.37.0 - Generator version: 7.6.0 - Build package: `org.openapitools.codegen.languages.RustClientCodegen` diff --git a/tools/unitctl/unitctl/Cargo.toml b/tools/unitctl/unitctl/Cargo.toml index a70d47d42..b4048b5ca 100644 --- a/tools/unitctl/unitctl/Cargo.toml +++ b/tools/unitctl/unitctl/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "unitctl" description = "CLI interface to the FreeUnit Control API" -version = "1.36.1" +version = "1.37.0" authors = ["Elijah Zupancic"] edition = "2021" rust-version = "1.95" diff --git a/version b/version index f702395d0..cf548338b 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ # Copyright (C) FreeUnit Community -NXT_VERSION=1.36.1 -NXT_VERNUM=13601 +NXT_VERSION=1.37.0 +NXT_VERNUM=13700 From 9627a35859b10fb0cf3b2dedd59a872719cafeaa Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 12:40:06 +0000 Subject: [PATCH 375/391] docs: say how CHANGES and the unitctl tag are really made "make -C docs/ changes" does not produce the CHANGES in the tree: it titles the release "Changes with Unit" and wraps every entry differently, because the file has been kept by hand, one changelog commit at a time. Following the documented step would rewrite the whole file. The unitctl tag is created by unitctl.yml when the release tag is pushed; pushing it by hand first makes that release job fail. Say so, and name the Docker Hub secrets release-docker.yml needs before the tag. --- RELEASE-PROCESS.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/RELEASE-PROCESS.md b/RELEASE-PROCESS.md index d4f2e5185..3aad4ee7d 100644 --- a/RELEASE-PROCESS.md +++ b/RELEASE-PROCESS.md @@ -44,11 +44,18 @@ Create a commit that updates the docs/changes.xml for this release. As well as adding the various entries also update the 'date' and 'time' fields. -## Generate the CHANGES file +## Update the CHANGES file - $ make -C docs/ changes && mv build/CHANGES . +CHANGES is maintained by hand: every changelog commit adds its entry to +both docs/changes.xml and CHANGES. For the release, retitle the top block +and set its date, keeping the leading blank line and the column the date +sits in: -See 24ed91f40 for an example. + Changes with FreeUnit X.Y.Z DD Mon YYYY + +Do not regenerate it with `make -C docs/ changes`: the generator titles the +block "Changes with Unit" and wraps every entry differently from the file +in the tree, so the result is a rewrite rather than an update. # Merge it @@ -70,6 +77,13 @@ The tag can be pushed just as the branch is. E.g. $ git push 1.33.0 +Do not create the `unitctl/X.Y.Z` tag by hand. Pushing `X.Y.Z` runs +.github/workflows/unitctl.yml, whose release job creates `unitctl/X.Y.Z` +and its release itself; a tag that already exists makes that step fail. +The same push runs build-deb.yml and release-docker.yml; the latter logs +in to Docker Hub, so the DOCKERHUB_USERNAME and DOCKERHUB_TOKEN secrets +have to be set before the tag is pushed. + # A new 'Release' From 6bb137e6c9b0a852bea0049537b45bc959b4cb48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 17:48:48 +0000 Subject: [PATCH 376/391] build(deps): Bump the minor-and-patch group across 7 directories with 21 updates Bumps the minor-and-patch group with 1 update in the /pkg/eol directory: [serde_json](https://github.com/serde-rs/json). Bumps the minor-and-patch group with 1 update in the /src/otel directory: [tokio](https://github.com/tokio-rs/tokio). Bumps the minor-and-patch group with 10 updates in the /src/wasm-wasi-component directory: | Package | From | To | | --- | --- | --- | | [tokio](https://github.com/tokio-rs/tokio) | `1.52.3` | `1.53.1` | | [anyhow](https://github.com/dtolnay/anyhow) | `1.0.103` | `1.0.104` | | [bytes](https://github.com/tokio-rs/bytes) | `1.12.0` | `1.12.1` | | [futures](https://github.com/rust-lang/futures-rs) | `0.3.32` | `0.3.34` | | [futures-util](https://github.com/rust-lang/futures-rs) | `0.3.32` | `0.3.34` | | [http](https://github.com/hyperium/http) | `1.4.2` | `1.5.0` | | [http-body](https://github.com/hyperium/http-body) | `1.0.1` | `1.1.0` | | [http-body-util](https://github.com/hyperium/http-body) | `0.1.3` | `0.1.5` | | [hyper](https://github.com/hyperium/hyper) | `1.10.1` | `1.11.1` | | [cc](https://github.com/rust-lang/cc-rs) | `1.2.66` | `1.5.1` | Bumps the minor-and-patch group with 2 updates in the /test/fake_otlp directory: [tokio](https://github.com/tokio-rs/tokio) and [prost](https://github.com/tokio-rs/prost). Bumps the minor-and-patch group with 1 update in the /test/wasm_component/hello_world directory: [bitflags](https://github.com/bitflags/bitflags). Bumps the minor-and-patch group with 1 update in the /test/wasm_component/spin_loop directory: [bitflags](https://github.com/bitflags/bitflags). Bumps the minor-and-patch group with 13 updates in the /tools/unitctl directory: | Package | From | To | | --- | --- | --- | | [serde_json](https://github.com/serde-rs/json) | `1.0.150` | `1.0.151` | | [tokio](https://github.com/tokio-rs/tokio) | `1.52.4` | `1.53.1` | | [futures](https://github.com/rust-lang/futures-rs) | `0.3.32` | `0.3.34` | | [http-body-util](https://github.com/hyperium/http-body) | `0.1.4` | `0.1.5` | | [hyper](https://github.com/hyperium/hyper) | `1.10.1` | `1.11.1` | | [serde](https://github.com/serde-rs/serde) | `1.0.228` | `1.0.229` | | [serde_derive](https://github.com/serde-rs/serde) | `1.0.228` | `1.0.229` | | [hyper-util](https://github.com/hyperium/hyper-util) | `0.1.20` | `0.1.21` | | [hyper-rustls](https://github.com/rustls/hyper-rustls) | `0.27.9` | `0.27.10` | | [which](https://github.com/harryfei/which-rs) | `8.0.5` | `8.0.6` | | [bollard](https://github.com/fussybeaver/bollard) | `0.21.0` | `0.21.1` | | [clap](https://github.com/clap-rs/clap) | `4.6.2` | `4.6.7` | | [rustls-pki-types](https://github.com/rustls/pki-types) | `1.15.0` | `1.15.1` | Updates `serde_json` from 1.0.149 to 1.0.151 - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.149...v1.0.151) Updates `tokio` from 1.52.3 to 1.53.1 - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.52.3...tokio-1.53.1) Updates `tokio` from 1.52.3 to 1.53.1 - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.52.3...tokio-1.53.1) Updates `anyhow` from 1.0.103 to 1.0.104 - [Release notes](https://github.com/dtolnay/anyhow/releases) - [Commits](https://github.com/dtolnay/anyhow/compare/1.0.103...1.0.104) Updates `bytes` from 1.12.0 to 1.12.1 - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.12.0...v1.12.1) Updates `futures` from 0.3.32 to 0.3.34 - [Release notes](https://github.com/rust-lang/futures-rs/releases) - [Changelog](https://github.com/rust-lang/futures-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/futures-rs/compare/0.3.32...0.3.34) Updates `futures-util` from 0.3.32 to 0.3.34 - [Release notes](https://github.com/rust-lang/futures-rs/releases) - [Changelog](https://github.com/rust-lang/futures-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/futures-rs/compare/0.3.32...0.3.34) Updates `http` from 1.4.2 to 1.5.0 - [Release notes](https://github.com/hyperium/http/releases) - [Changelog](https://github.com/hyperium/http/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/http/compare/v1.4.2...v1.5.0) Updates `http-body` from 1.0.1 to 1.1.0 - [Release notes](https://github.com/hyperium/http-body/releases) - [Commits](https://github.com/hyperium/http-body/compare/v1.0.1...v1.1.0) Updates `http-body-util` from 0.1.3 to 0.1.5 - [Release notes](https://github.com/hyperium/http-body/releases) - [Commits](https://github.com/hyperium/http-body/compare/http-body-util-v0.1.3...http-body-util-v0.1.5) Updates `hyper` from 1.10.1 to 1.11.1 - [Release notes](https://github.com/hyperium/hyper/releases) - [Changelog](https://github.com/hyperium/hyper/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper/compare/v1.10.1...v1.11.1) Updates `cc` from 1.2.66 to 1.5.1 - [Release notes](https://github.com/rust-lang/cc-rs/releases) - [Changelog](https://github.com/rust-lang/cc-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/cc-rs/compare/cc-v1.2.66...cc-v1.5.1) Updates `tokio` from 1.52.3 to 1.53.1 - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.52.3...tokio-1.53.1) Updates `prost` from 0.14.3 to 0.14.4 - [Release notes](https://github.com/tokio-rs/prost/releases) - [Changelog](https://github.com/tokio-rs/prost/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/prost/compare/v0.14.3...v0.14.4) Updates `bitflags` from 2.5.0 to 2.13.2 - [Release notes](https://github.com/bitflags/bitflags/releases) - [Changelog](https://github.com/bitflags/bitflags/blob/main/CHANGELOG.md) - [Commits](https://github.com/bitflags/bitflags/compare/2.5.0...2.13.2) Updates `bitflags` from 2.5.0 to 2.13.2 - [Release notes](https://github.com/bitflags/bitflags/releases) - [Changelog](https://github.com/bitflags/bitflags/blob/main/CHANGELOG.md) - [Commits](https://github.com/bitflags/bitflags/compare/2.5.0...2.13.2) Updates `serde_json` from 1.0.150 to 1.0.151 - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.149...v1.0.151) Updates `tokio` from 1.52.4 to 1.53.1 - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.52.3...tokio-1.53.1) Updates `futures` from 0.3.32 to 0.3.34 - [Release notes](https://github.com/rust-lang/futures-rs/releases) - [Changelog](https://github.com/rust-lang/futures-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-lang/futures-rs/compare/0.3.32...0.3.34) Updates `http-body-util` from 0.1.4 to 0.1.5 - [Release notes](https://github.com/hyperium/http-body/releases) - [Commits](https://github.com/hyperium/http-body/compare/http-body-util-v0.1.3...http-body-util-v0.1.5) Updates `hyper` from 1.10.1 to 1.11.1 - [Release notes](https://github.com/hyperium/hyper/releases) - [Changelog](https://github.com/hyperium/hyper/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper/compare/v1.10.1...v1.11.1) Updates `serde` from 1.0.228 to 1.0.229 - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.228...v1.0.229) Updates `serde_derive` from 1.0.228 to 1.0.229 - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.228...v1.0.229) Updates `hyper-util` from 0.1.20 to 0.1.21 - [Release notes](https://github.com/hyperium/hyper-util/releases) - [Changelog](https://github.com/hyperium/hyper-util/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper-util/compare/v0.1.20...v0.1.21) Updates `hyper-rustls` from 0.27.9 to 0.27.10 - [Release notes](https://github.com/rustls/hyper-rustls/releases) - [Commits](https://github.com/rustls/hyper-rustls/compare/v/0.27.9...v/0.27.10) Updates `which` from 8.0.5 to 8.0.6 - [Release notes](https://github.com/harryfei/which-rs/releases) - [Changelog](https://github.com/harryfei/which-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/harryfei/which-rs/compare/8.0.5...8.0.6) Updates `bollard` from 0.21.0 to 0.21.1 - [Release notes](https://github.com/fussybeaver/bollard/releases) - [Changelog](https://github.com/fussybeaver/bollard/blob/master/RELEASE.md) - [Commits](https://github.com/fussybeaver/bollard/compare/v0.21.0...v0.21.1) Updates `clap` from 4.6.2 to 4.6.7 - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/main/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.6.2...clap_complete-v4.6.7) Updates `rustls-pki-types` from 1.15.0 to 1.15.1 - [Release notes](https://github.com/rustls/pki-types/releases) - [Commits](https://github.com/rustls/pki-types/compare/v/1.15.0...v/1.15.1) --- updated-dependencies: - dependency-name: serde_json dependency-version: 1.0.151 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: tokio dependency-version: 1.53.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: tokio dependency-version: 1.53.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: anyhow dependency-version: 1.0.104 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: bytes dependency-version: 1.12.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: futures dependency-version: 0.3.34 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: futures-util dependency-version: 0.3.34 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: http dependency-version: 1.5.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: http-body dependency-version: 1.1.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: http-body-util dependency-version: 0.1.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: hyper dependency-version: 1.11.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: cc dependency-version: 1.5.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: tokio dependency-version: 1.53.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: prost dependency-version: 0.14.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: bitflags dependency-version: 2.13.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: bitflags dependency-version: 2.13.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: serde_json dependency-version: 1.0.151 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: tokio dependency-version: 1.53.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: futures dependency-version: 0.3.34 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: http-body-util dependency-version: 0.1.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: hyper dependency-version: 1.11.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: minor-and-patch - dependency-name: serde dependency-version: 1.0.229 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: serde_derive dependency-version: 1.0.229 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: hyper-util dependency-version: 0.1.21 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: hyper-rustls dependency-version: 0.27.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: which dependency-version: 8.0.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: bollard dependency-version: 0.21.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: clap dependency-version: 4.6.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: rustls-pki-types dependency-version: 1.15.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch ... Signed-off-by: dependabot[bot] --- pkg/eol/Cargo.lock | 20 +-- src/otel/Cargo.lock | 53 +++---- src/wasm-wasi-component/Cargo.lock | 151 ++++++++++---------- test/fake_otlp/Cargo.lock | 37 ++--- test/wasm_component/hello_world/Cargo.lock | 6 +- test/wasm_component/spin_loop/Cargo.lock | 6 +- tools/unitctl/Cargo.lock | 152 +++++++++++---------- 7 files changed, 228 insertions(+), 197 deletions(-) diff --git a/pkg/eol/Cargo.lock b/pkg/eol/Cargo.lock index 19e2a6f70..93fc6d1b4 100644 --- a/pkg/eol/Cargo.lock +++ b/pkg/eol/Cargo.lock @@ -34,27 +34,27 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", ] [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", @@ -63,9 +63,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -76,9 +76,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", diff --git a/src/otel/Cargo.lock b/src/otel/Cargo.lock index 5122b5f62..db0b90b3c 100644 --- a/src/otel/Cargo.lock +++ b/src/otel/Cargo.lock @@ -74,11 +74,17 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bumpalo" @@ -88,9 +94,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cfg-if" @@ -247,9 +253,9 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "http" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -257,9 +263,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -267,9 +273,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -292,9 +298,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -327,16 +333,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "ipnet", "libc", @@ -707,9 +714,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -717,9 +724,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -787,7 +794,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -938,9 +945,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -994,7 +1001,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http", diff --git a/src/wasm-wasi-component/Cargo.lock b/src/wasm-wasi-component/Cargo.lock index 72dd6f7a0..41403cfbd 100644 --- a/src/wasm-wasi-component/Cargo.lock +++ b/src/wasm-wasi-component/Cargo.lock @@ -34,9 +34,9 @@ checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" @@ -52,7 +52,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -78,14 +78,14 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.3.0", - "syn", + "syn 2.0.118", ] [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "block-buffer" @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cap-primitives" @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.66" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", "shlex 2.0.1", @@ -395,7 +395,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -443,9 +443,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "fnv" @@ -481,9 +481,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -496,9 +496,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -506,15 +506,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -523,38 +523,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -651,9 +651,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -661,9 +661,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -671,9 +671,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -696,9 +696,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -1053,7 +1053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.118", ] [[package]] @@ -1085,7 +1085,7 @@ checksum = "864f92c9538de48325a4f8709ec83e56471446798d48a29073ca094c5447b549" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1105,9 +1105,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom", @@ -1137,9 +1137,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1149,9 +1149,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1211,9 +1211,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1221,29 +1221,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1317,6 +1317,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1325,7 +1336,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1369,7 +1380,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1380,7 +1391,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1395,9 +1406,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -1439,7 +1450,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1655,7 +1666,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasmtime-internal-component-util", "wasmtime-internal-wit-bindgen", "wit-parser", @@ -1760,7 +1771,7 @@ checksum = "fca407e3e2809c0e6568be132065095a16f184c30c6bee4157dfdc497aee10a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1868,7 +1879,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasmtime-environ", "witx", ] @@ -1881,7 +1892,7 @@ checksum = "488783149d44017c9807189b16cc09e02e84059740fe64178b5ab3cc19e9fc0b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wiggle-generate", ] @@ -2141,7 +2152,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2162,7 +2173,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2196,7 +2207,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/test/fake_otlp/Cargo.lock b/test/fake_otlp/Cargo.lock index ba7a103c0..c8d8cc42f 100644 --- a/test/fake_otlp/Cargo.lock +++ b/test/fake_otlp/Cargo.lock @@ -82,9 +82,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cfg-if" @@ -222,9 +222,9 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "http" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -267,9 +267,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -302,15 +302,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "libc", "pin-project-lite", @@ -498,9 +499,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -508,9 +509,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -650,9 +651,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", diff --git a/test/wasm_component/hello_world/Cargo.lock b/test/wasm_component/hello_world/Cargo.lock index 2daeb73de..3bf1dd23b 100644 --- a/test/wasm_component/hello_world/Cargo.lock +++ b/test/wasm_component/hello_world/Cargo.lock @@ -1,12 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "bitflags" -version = "2.5.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "test-wasi-component" diff --git a/test/wasm_component/spin_loop/Cargo.lock b/test/wasm_component/spin_loop/Cargo.lock index 2daeb73de..3bf1dd23b 100644 --- a/test/wasm_component/spin_loop/Cargo.lock +++ b/test/wasm_component/spin_loop/Cargo.lock @@ -1,12 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "bitflags" -version = "2.5.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "test-wasi-component" diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 63953a0a3..118041a3a 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -98,15 +98,15 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bollard" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d0a013e3d3ee4edd61e779adf117944c08902d375f18630a0c5b8f95659734" +checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" dependencies = [ "base64", "bollard-stubs", @@ -153,9 +153,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.67" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "jobserver", @@ -171,9 +171,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "clap" -version = "4.6.2" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -181,9 +181,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.2" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -193,14 +193,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -290,7 +290,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -333,9 +333,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "form_urlencoded" @@ -354,9 +354,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -369,9 +369,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -379,15 +379,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -396,38 +396,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -494,9 +494,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -514,9 +514,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -539,9 +539,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -575,9 +575,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http", "hyper", @@ -593,15 +593,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "libc", "pin-project-lite", @@ -1077,9 +1078,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "zeroize", ] @@ -1145,9 +1146,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1155,29 +1156,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", @@ -1195,7 +1196,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1267,6 +1268,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1275,7 +1287,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1334,7 +1346,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1349,9 +1361,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.4" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317fafbbe3f02fc663dad00ea6186197de963cd4190e86a26d8d0fae095539af" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -1370,7 +1382,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1572,9 +1584,9 @@ dependencies = [ [[package]] name = "which" -version = "8.0.5" +version = "8.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" dependencies = [ "libc", ] @@ -1663,7 +1675,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1674,7 +1686,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1849,7 +1861,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -1870,7 +1882,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1890,7 +1902,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -1930,7 +1942,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] From f0f589ee1709d7ed6c45efe17291f3cb79a1f5e6 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 20:12:17 +0200 Subject: [PATCH 377/391] ci: keep the otel crates out of the Dependabot minor-and-patch group The first Dependabot run did not group the otel crates. Dependabot keeps a dependency only in its most specific group, and a group without "patterns" scores higher than the pattern "opentelemetry*". So the minor-and-patch group claimed opentelemetry, opentelemetry_sdk, opentelemetry-otlp and opentelemetry-proto. Cargo treats 0.32 -> 0.33 as a major update, so that group rejected them, and each crate fell through to its own pull request. The first one, https://github.com/freeunitorg/freeunit/pull/569, does not build: the lock then holds two opentelemetry versions. Exclude opentelemetry*, tonic* and prost* from minor-and-patch, so these crates are only in the otel group, which has no update-types and moves them together. The run also showed that a group is one pull request for all directories. Fix the comment that said otherwise. How open-pull-requests-limit counts with several directories is not clear: this run looked block-wide, while https://github.com/dependabot/dependabot-core/issues/10395 shows a per-directory count for ungrouped updates. So put every version update in a group: a new "major" group takes the major updates (including a 0.x minor, which cargo rates as major), with the same otel exclusion. At most three version-update pull requests are then open either way, and the limit is 3. See updater/lib/dependabot/updater/pattern_specificity_calculator.rb and common/lib/dependabot/dependency_group.rb in https://github.com/dependabot/dependabot-core. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- .github/dependabot.yml | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c7a01a924..8e93cf8c3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -23,15 +23,15 @@ updates: - "/test/wasm_component/spin_loop" schedule: interval: weekly - # The limit counts per directory, not for the whole block - # (https://github.com/dependabot/dependabot-core/issues/10395). With the - # groups below, a directory needs one or two pull requests a week. - open-pull-requests-limit: 2 + # Every version update below belongs to a group, and a group is one + # pull request for all directories. So at most three version-update + # pull requests are open, whether the limit counts per directory or for + # the whole block. + open-pull-requests-limit: 3 commit-message: prefix: "build(deps)" - # A group opens one pull request per directory, so test/fake_otlp gets - # its own otel pull request. "group-by: dependency-name" would join the - # directories but split the otel crates, which must move together. + # A group is one pull request for all directories, so src/otel and + # test/fake_otlp move together in one otel pull request. groups: otel: patterns: @@ -46,10 +46,26 @@ updates: - "opentelemetry*" - "tonic*" - "prost*" + # A group without "patterns" outranks "opentelemetry*" when Dependabot + # picks the most specific group, so the two groups below must exclude + # the otel crates. Otherwise they claim them and split the stack. minor-and-patch: + exclude-patterns: + - "opentelemetry*" + - "tonic*" + - "prost*" update-types: - "minor" - "patch" + # Major updates, including a 0.x minor, which cargo rates as major. + # They come together in one pull request. + major: + exclude-patterns: + - "opentelemetry*" + - "tonic*" + - "prost*" + update-types: + - "major" ignore: # Wasmtime is updated by hand. Its version is also pinned in # pkg/contrib/src/wasmtime/version, EOL.md and pkg/eol.json, which a From ccf1f52373bc5b31b7e3b6d046f854251b515c04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:31:41 +0000 Subject: [PATCH 378/391] build(deps): Bump the otel group across 2 directories with 5 updates Bumps the otel group with 4 updates in the /src/otel directory: [opentelemetry](https://github.com/open-telemetry/opentelemetry-rust), [opentelemetry_sdk](https://github.com/open-telemetry/opentelemetry-rust), [opentelemetry-otlp](https://github.com/open-telemetry/opentelemetry-rust) and [opentelemetry-semantic-conventions](https://github.com/open-telemetry/opentelemetry-rust). Bumps the otel group with 1 update in the /test/fake_otlp directory: [opentelemetry-proto](https://github.com/open-telemetry/opentelemetry-rust). Updates `opentelemetry` from 0.32.0 to 0.33.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-rust/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-rust/blob/opentelemetry-0.33.0/docs/release_0.33.md) - [Commits](https://github.com/open-telemetry/opentelemetry-rust/compare/opentelemetry-0.32.0...opentelemetry-0.33.0) Updates `opentelemetry_sdk` from 0.32.1 to 0.33.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-rust/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-rust/blob/opentelemetry_sdk-0.33.0/docs/release_0.33.md) - [Commits](https://github.com/open-telemetry/opentelemetry-rust/compare/opentelemetry-semantic-conventions-0.32.1...opentelemetry_sdk-0.33.0) Updates `opentelemetry-otlp` from 0.32.0 to 0.33.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-rust/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-rust/blob/opentelemetry-otlp-0.33.0/docs/release_0.33.md) - [Commits](https://github.com/open-telemetry/opentelemetry-rust/compare/opentelemetry-otlp-0.32.0...opentelemetry-otlp-0.33.0) Updates `opentelemetry-semantic-conventions` from 0.32.0 to 0.33.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-rust/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-rust/blob/opentelemetry-semantic-conventions-0.33.0/docs/release_0.33.md) - [Commits](https://github.com/open-telemetry/opentelemetry-rust/compare/opentelemetry-semantic-conventions-0.32.0...opentelemetry-semantic-conventions-0.33.0) Updates `opentelemetry-proto` from 0.32.0 to 0.33.0 - [Release notes](https://github.com/open-telemetry/opentelemetry-rust/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-rust/blob/opentelemetry-proto-0.33.0/docs/release_0.33.md) - [Commits](https://github.com/open-telemetry/opentelemetry-rust/compare/opentelemetry-proto-0.32.0...opentelemetry-proto-0.33.0) --- updated-dependencies: - dependency-name: opentelemetry dependency-version: 0.33.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: otel - dependency-name: opentelemetry_sdk dependency-version: 0.33.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: otel - dependency-name: opentelemetry-otlp dependency-version: 0.33.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: otel - dependency-name: opentelemetry-semantic-conventions dependency-version: 0.33.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: otel - dependency-name: opentelemetry-proto dependency-version: 0.33.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: otel ... Signed-off-by: dependabot[bot] --- src/otel/Cargo.lock | 25 +++++++++++++------------ src/otel/Cargo.toml | 8 ++++---- test/fake_otlp/Cargo.lock | 12 ++++++------ test/fake_otlp/Cargo.toml | 2 +- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/otel/Cargo.lock b/src/otel/Cargo.lock index db0b90b3c..a52cfc5f5 100644 --- a/src/otel/Cargo.lock +++ b/src/otel/Cargo.lock @@ -556,9 +556,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "opentelemetry" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" dependencies = [ "futures-core", "futures-sink", @@ -570,9 +570,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" +checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" dependencies = [ "async-trait", "bytes", @@ -583,11 +583,12 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" dependencies = [ "http", + "httpdate", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -602,9 +603,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -615,15 +616,15 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" +checksum = "d288b1e1bc3590052fbb8f0f635ef248dd9022cf1fbaeb6582b7ad6081a17b87" [[package]] name = "opentelemetry_sdk" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" dependencies = [ "futures-channel", "futures-executor", diff --git a/src/otel/Cargo.toml b/src/otel/Cargo.toml index 86ab6afd1..f3b47e1be 100644 --- a/src/otel/Cargo.toml +++ b/src/otel/Cargo.toml @@ -25,14 +25,14 @@ bench = false # staticlib), so both transports are plaintext — the endpoint must be plain # http:// (h2c for grpc). [dependencies] -opentelemetry = "0.32" -opentelemetry_sdk = { version = "0.32", features = ["trace"] } -opentelemetry-otlp = { version = "0.32", default-features = false, features = [ +opentelemetry = "0.33" +opentelemetry_sdk = { version = "0.33", features = ["trace"] } +opentelemetry-otlp = { version = "0.33", default-features = false, features = [ "trace", "http-proto", "reqwest-blocking-client", "grpc-tonic", ] } -opentelemetry-semantic-conventions = "0.32" +opentelemetry-semantic-conventions = "0.33" tonic = "0.14" tokio = { version = "1", features = ["rt-multi-thread"] } diff --git a/test/fake_otlp/Cargo.lock b/test/fake_otlp/Cargo.lock index c8d8cc42f..07f80e752 100644 --- a/test/fake_otlp/Cargo.lock +++ b/test/fake_otlp/Cargo.lock @@ -401,9 +401,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "opentelemetry" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" dependencies = [ "futures-core", "futures-sink", @@ -414,9 +414,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -427,9 +427,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" dependencies = [ "futures-channel", "futures-executor", diff --git a/test/fake_otlp/Cargo.toml b/test/fake_otlp/Cargo.toml index 5c8549c8a..6d8a5f2b4 100644 --- a/test/fake_otlp/Cargo.toml +++ b/test/fake_otlp/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" tonic = "0.14" prost = "0.14" tokio = { version = "1", features = ["rt-multi-thread", "macros"] } -opentelemetry-proto = { version = "0.32", default-features = false, features = [ +opentelemetry-proto = { version = "0.33", default-features = false, features = [ "gen-tonic", "trace", ] } From 47d2f933a6324ff736386f6df9c07463ae031614 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 22:11:31 +0200 Subject: [PATCH 379/391] changelog: open the 1.37.1 section 1.37.0 is tagged. Changes merged from now on go into a new 1.37.1 block: the CHANGES header, a "unit" stanza in docs/changes.xml with no entries yet (changes.dtd allows that), and the per-module stanza that says "FreeUnit updated to 1.37.1". The date is the opening date; the release commit sets the real one, as it did for 1.37.0. The "version" file stays at 1.37.0 until the release commit. Until then a build-deb dispatch on master sees 1.37.0 in "version" and 1.37.1 in the top changes.xml stanza, as during the 1.36.2 period. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 3 +++ docs/changes.xml | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/CHANGES b/CHANGES index be7818c9b..a9235c4c8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,7 @@ +Changes with FreeUnit 1.37.1 04 Oct 2026 + + Changes with FreeUnit 1.37.0 04 Oct 2026 *) Bugfix: a chunked request body that grew over "max_body_size" diff --git a/docs/changes.xml b/docs/changes.xml index 7a5494088..bfc8f35fa 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -5,6 +5,37 @@ + + + + + + + + + +FreeUnit updated to 1.37.1. + + + + + + Date: Sun, 4 Oct 2026 22:31:22 +0200 Subject: [PATCH 380/391] otel: stop requiring and linking OpenSSL auto/otel tested for OpenSSL on every system except macOS. Without OpenSSL, configure stopped with "OpenTelemetry support requires OpenSSL." With OpenSSL, configure added "-lssl -lcrypto" to the link of unitd and to EXTRA_LIBS in unit.pc. The test came with the first OTel build support in https://github.com/nginx/unit/commit/9d3dcb800aba0c036b032ccd00197712c3f5d0d9. At that time, src/otel used reqwest 0.12.7 with its default features. They include native-tls, and native-tls links openssl-sys. https://github.com/freeunitorg/freeunit/commit/de24c384d4c4b3189d00d51358a05bbcf979a1f2 removed native-tls and openssl-sys from src/otel/Cargo.lock. Since 1.36.0, the crate links no TLS stack. src/nxt_otel.c calls no OpenSSL function. In src/otel, "cargo rustc --release -- --print native-static-libs" prints "-lgcc_s -lutil -lrt -lpthread -lm -ldl -lc". The test was a leftover. It had two effects on a build with --otel and without --openssl: - On a host without the OpenSSL headers, configure failed. - On a host with the headers, unitd got NEEDED entries for libssl and libcrypto when the linker kept unused libraries. On AlmaLinux 8 (gcc 8.5.0), unitd needed libssl.so.1.1 and libcrypto.so.1.1. Application processes are forked from unitd, so they had these libraries loaded too. The gcc of Ubuntu 24.04 and Debian 13 passes --as-needed, so the entries do not appear there. Now auto/otel defines NXT_HAVE_OTEL without a test. NXT_OTEL_LIBS is empty on every system except macOS. The macOS libraries do not change. On Linux, NXT_LIBM, NXT_LIBS and the compiler driver give the libraries that the crate needs. The link succeeds without -lutil on glibc 2.28 and 2.39. The core job in build-test.yml now runs "./configure --otel" before the real configure. The step fails if build/Makefile has -lssl or -lcrypto. It reads the Makefile because the gcc of the runner passes --as-needed, and readelf on unitd would not show the fault. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-test.yml | 15 ++++++++++++ CHANGES | 6 +++++ auto/otel | 39 ++++++++------------------------ docs/changes.xml | 10 ++++++++ 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f355784f4..365610709 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -192,6 +192,21 @@ jobs: restore-keys: | otel-cargo-${{ runner.os }}-${{ runner.arch }}-${{ steps.rustc.outputs.version }}- + # An --otel build without --openssl must not link OpenSSL. The + # src/otel crate links no TLS stack. This check reads the link + # line that configure writes. readelf on unitd cannot show the + # fault here: the gcc of this runner passes --as-needed, and that + # drops an unused -lssl. The next step runs configure again, so + # this step removes its output. + - name: Check that --otel without --openssl links no OpenSSL + run: | + ./configure --otel + if grep -n -E -- '-l(ssl|crypto)' build/Makefile; then + echo "::error::an --otel build without --openssl links OpenSSL" + exit 1 + fi + rm -rf build Makefile + - name: Configure unit # LD_LIBRARY_PATH is set only for this step so that auto/feature # test binaries load OpenSSL 3.6 at runtime. It is NOT exported diff --git a/CHANGES b/CHANGES index a9235c4c8..bdb703de6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,12 @@ Changes with FreeUnit 1.37.1 04 Oct 2026 + *) Change: "--otel" no longer needs OpenSSL. Before, configure + stopped with "OpenTelemetry support requires OpenSSL." when the + OpenSSL headers were missing, and it linked unitd with OpenSSL + even without "--openssl". The OTel exporter has linked no TLS + stack since 1.36.0. macOS builds did not have the problem. + Changes with FreeUnit 1.37.0 04 Oct 2026 diff --git a/auto/otel b/auto/otel index 4d059a285..9a75e7dfc 100644 --- a/auto/otel +++ b/auto/otel @@ -19,44 +19,25 @@ if [ $NXT_OTEL = YES ]; then fi $echo "found" + # The src/otel crate links no TLS stack, so it needs no OpenSSL. + # On Linux, its other libraries come from NXT_LIBM, NXT_LIBS and + # the compiler driver (libc, libgcc_s). + case "$NXT_SYSTEM" in Darwin) NXT_OTEL_LIBS="-framework CoreFoundation -framework Security -framework SystemConfiguration" + ;; + *) + NXT_OTEL_LIBS= + ;; + esac - cat << END >> $NXT_AUTO_CONFIG_H + cat << END >> $NXT_AUTO_CONFIG_H #ifndef NXT_HAVE_OTEL #define NXT_HAVE_OTEL 1 #endif END - ;; - *) - - $echo -n " - " - - nxt_feature="OpenSSL library" - nxt_feature_name=NXT_HAVE_OTEL - nxt_feature_run=yes - nxt_feature_incs= - nxt_feature_libs="-lssl -lcrypto" - nxt_feature_test="#include - - int main(void) { - SSL_library_init(); - return 0; - }" - . auto/feature - - if [ ! $nxt_found = yes ]; then - $echo - $echo $0: error: OpenTelemetry support requires OpenSSL. - $echo - exit 1; - fi - - NXT_OTEL_LIBS="-lssl -lcrypto" - ;; - esac fi diff --git a/docs/changes.xml b/docs/changes.xml index bfc8f35fa..3781d707d 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,16 @@ date="2026-10-04" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +"--otel" no longer needs OpenSSL. Before, configure stopped with +"OpenTelemetry support requires OpenSSL." when the OpenSSL headers were +missing, and it linked unitd with OpenSSL even without "--openssl". The OTel +exporter has linked no TLS stack since 1.36.0. macOS builds did not have the +problem. + + + From a7e3a2981ca7f6ed4a5da8c7f9a71bdb34890f88 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 16:09:53 +0200 Subject: [PATCH 381/391] wasm: refuse a handler export that is not a function nxt_wasmtime_get_function_exports() looked up each configured handler name with wasmtime_linker_get(). It checked only that the name existed, and then read item.of.func for every kind of export. The worker later called that value as a function. For a global or a memory, the value holds no function reference, and Wasmtime does not define the call. With Wasmtime 48.0.5 the worker died on signal 11 when the host called the malloc handler. The lookup now also checks that the kind is WASMTIME_EXTERN_FUNC. The lookup of "memory" has checked WASMTIME_EXTERN_MEMORY in the same way since https://github.com/freeunitorg/freeunit/pull/554. The worker logs "module export () is not a function" and exits with code 1. The router answers 503. A new test sets the malloc handler to "__heap_base", a global, and to "memory", a memory. Both exports are in the hello guest. On the old code the worker died on signal 11, and the log line was missing. This was found in the review of https://github.com/freeunitorg/freeunit/pull/554#issuecomment-5980129553. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 5 +++++ docs/changes.xml | 9 +++++++++ src/wasm/nxt_rt_wasmtime.c | 8 ++++++++ test/test_wasm.py | 11 +++++++++++ 4 files changed, 33 insertions(+) diff --git a/CHANGES b/CHANGES index bdb703de6..127146765 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,11 @@ Changes with FreeUnit 1.37.1 04 Oct 2026 even without "--openssl". The OTel exporter has linked no TLS stack since 1.36.0. macOS builds did not have the problem. + *) Bugfix: the wasm module took any export with a handler name as a + function. If that export was not a function, such as a global or + a memory, the worker crashed on the first call of the handler. The + module now refuses such a guest when the worker starts. + Changes with FreeUnit 1.37.0 04 Oct 2026 diff --git a/docs/changes.xml b/docs/changes.xml index 3781d707d..492b4c9ac 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -20,6 +20,15 @@ problem. + + +the wasm module took any export with a handler name as a function. If that +export was not a function, such as a global or a memory, the worker crashed on +the first call of the handler. The module now refuses such a guest when the +worker starts. + + + diff --git a/src/wasm/nxt_rt_wasmtime.c b/src/wasm/nxt_rt_wasmtime.c index 81b478e22..1511524c5 100644 --- a/src/wasm/nxt_rt_wasmtime.c +++ b/src/wasm/nxt_rt_wasmtime.c @@ -281,6 +281,14 @@ nxt_wasmtime_get_function_exports(nxt_wasm_ctx_t *ctx) ctx->fh[i].func_name); return -1; } + + if (item.kind != WASMTIME_EXTERN_FUNC) { + nxt_wasmtime_err_msg(NULL, NULL, + "module export (%s) is not a function", + ctx->fh[i].func_name); + return -1; + } + ctx->fh[i].func = item.of.func; } diff --git a/test/test_wasm.py b/test/test_wasm.py index ca7935290..783630866 100644 --- a/test/test_wasm.py +++ b/test/test_wasm.py @@ -104,3 +104,14 @@ def test_wasm_malloc_unaligned(wait_for_record): record = r'malloc handler returned offset \d+ that is not aligned' assert wait_for_record(record) is not None + + +# The guest exports "__heap_base" as a global and "memory" as a memory. +@pytest.mark.parametrize('handler', ['__heap_base', 'memory']) +def test_wasm_handler_not_function(handler, wait_for_record): + client.load('hello', malloc_handler=handler) + + assert client.get()['status'] == 503 + + record = rf'module export \({handler}\) is not a function' + assert wait_for_record(record) is not None From 5c785aaee8686e6316b7063fae9193d280f398ef Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Sun, 4 Oct 2026 16:10:16 +0200 Subject: [PATCH 382/391] wasm: trap an import that needs a request before the first request The guest can run code before the host serves a request. wasmtime_linker_module() runs the start function of the guest, and "_initialize" for a reactor. nxt_wasmtime_init_memory() calls the malloc handler before it stores the memory handle. nxt_wasm_start() calls the module_init hook before the first request. In all of these, ctx->req is NULL. nxt_wasm_send_response(), nxt_wasm_send_headers() and nxt_wasm_response_end() then crashed the worker: - Before the memory handle was set, send_response and send_headers passed a zeroed wasmtime_memory_t to wasmtime_memory_data(). Wasmtime 48.0.5 panicked with "object used with the wrong store", and the worker died on signal 6. - response_end called nxt_unit_request_done() with a NULL request, and the worker died on signal 11. In the module_init hook the prototype started a new worker after each crash: 204 "app process ... exited on signal 11" alerts in one test, and the request got no response. These three imports now return a trap when ctx->req is NULL. The host serves a request only after nxt_wasmtime_init() succeeds, so a request also implies a memory handle. A trap in the start function, "_initialize" or the malloc handler fails the startup, and the router answers 503. A trap in a hook ends the hook, and the worker serves requests as before. Refusing a 64-bit memory before instantiation does not cover these cases. A wasm32 guest reaches the same code, and response_end does not use the memory. The comment on nxt_wasmtime_update_baddr() now says that the base is valid only until the guest runs again. A new guest, test/wasm/noreq/noreq.c, is built without libc. It calls one of the three imports from "_initialize", from a malloc handler or from a module_init hook. clang has no attribute for the wasm start section, so the test uses "_initialize", which runs in the same wasmtime_linker_module() call. A hand-encoded module whose start function is the import gave the same result for all three imports. Each test fails on the old code, and it also fails when the check is removed from the import that it calls. These cases were found in the review of https://github.com/freeunitorg/freeunit/pull/554#issuecomment-5980129553. ctx->req still keeps a finished request after the request ends. That is not changed here. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 Reviewed-by: DeepSeek V4.1 Flash --- CHANGES | 8 +++ docs/changes.xml | 11 ++++ src/wasm/nxt_rt_wasmtime.c | 47 +++++++++++++++- test/test_wasm_imports.py | 107 +++++++++++++++++++++++++++++++++++++ test/wasm/noreq/noreq.c | 97 +++++++++++++++++++++++++++++++++ 5 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 test/test_wasm_imports.py create mode 100644 test/wasm/noreq/noreq.c diff --git a/CHANGES b/CHANGES index 127146765..225c8c3a4 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,14 @@ Changes with FreeUnit 1.37.1 04 Oct 2026 even without "--openssl". The OTel exporter has linked no TLS stack since 1.36.0. macOS builds did not have the problem. + *) Bugfix: the worker of a wasm application crashed when the guest + called the import nxt_wasm_send_response, nxt_wasm_send_headers or + nxt_wasm_response_end before the first request. This could happen + in the start function or "_initialize" of the guest, in the malloc + handler, or in the module_init hook. These imports now trap when + the worker has no request yet. The worker then does not start, + except in the hook, where the trap ends only the hook. + *) Bugfix: the wasm module took any export with a handler name as a function. If that export was not a function, such as a global or a memory, the worker crashed on the first call of the handler. The diff --git a/docs/changes.xml b/docs/changes.xml index 492b4c9ac..06ffa7937 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -20,6 +20,17 @@ problem. + + +the worker of a wasm application crashed when the guest called the import +nxt_wasm_send_response, nxt_wasm_send_headers or nxt_wasm_response_end before +the first request. This could happen in the start function or "_initialize" +of the guest, in the malloc handler, or in the module_init hook. These +imports now trap when the worker has no request yet. The worker then does +not start, except in the hook, where the trap ends only the hook. + + + the wasm module took any export with a handler name as a function. If that diff --git a/src/wasm/nxt_rt_wasmtime.c b/src/wasm/nxt_rt_wasmtime.c index 1511524c5..6d9e401f1 100644 --- a/src/wasm/nxt_rt_wasmtime.c +++ b/src/wasm/nxt_rt_wasmtime.c @@ -61,7 +61,7 @@ nxt_wasmtime_err_msg(wasmtime_error_t *error, wasm_trap_t *trap, /* * memory.grow can move the linear memory to a new address. Read the base * again after the guest has run, and before an import function reads or - * writes the memory. + * writes the memory. The value is valid only until the guest runs again. */ static void nxt_wasmtime_update_baddr(nxt_wasm_ctx_t *ctx, wasmtime_context_t *wctx) @@ -71,6 +71,27 @@ nxt_wasmtime_update_baddr(nxt_wasm_ctx_t *ctx, wasmtime_context_t *wctx) } +/* + * The guest can call an import before the host serves a request: from its + * start function or "_initialize" in wasmtime_linker_module(), from the + * malloc handler, or from the module_init hook. The host has no request to + * answer then. Before nxt_wasmtime_init_memory() it also has no handle for + * the memory. An import that needs a request traps in that case. The host + * serves a request only after nxt_wasmtime_init() succeeds, so a request + * implies a memory handle. The host does not clear ctx->req after a + * request, so this check does not show that a request is still live. + */ +static wasm_trap_t * +nxt_wasmtime_check_request(const nxt_wasm_ctx_t *ctx, const char *msg) +{ + if (ctx->req != NULL) { + return NULL; + } + + return wasmtime_trap_new(msg, strlen(msg)); +} + + static wasm_trap_t * nxt_wasm_get_init_mem_size(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, @@ -87,6 +108,14 @@ nxt_wasm_response_end(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, wasmtime_val_t *results, size_t nresults) { + wasm_trap_t *trap; + + trap = nxt_wasmtime_check_request(env, "nxt_wasm_response_end() called " + "outside a request"); + if (trap != NULL) { + return trap; + } + nxt_wasm_do_response_end(env); return NULL; @@ -98,6 +127,14 @@ nxt_wasm_send_response(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, wasmtime_val_t *results, size_t nresults) { + wasm_trap_t *trap; + + trap = nxt_wasmtime_check_request(env, "nxt_wasm_send_response() called " + "outside a request"); + if (trap != NULL) { + return trap; + } + nxt_wasmtime_update_baddr(env, wasmtime_caller_context(caller)); nxt_wasm_do_send_response(env, args[0].of.i32); @@ -111,6 +148,14 @@ nxt_wasm_send_headers(void *env, wasmtime_caller_t *caller, const wasmtime_val_t *args, size_t nargs, wasmtime_val_t *results, size_t nresults) { + wasm_trap_t *trap; + + trap = nxt_wasmtime_check_request(env, "nxt_wasm_send_headers() called " + "outside a request"); + if (trap != NULL) { + return trap; + } + nxt_wasmtime_update_baddr(env, wasmtime_caller_context(caller)); nxt_wasm_do_send_headers(env, args[0].of.i32); diff --git a/test/test_wasm_imports.py b/test/test_wasm_imports.py new file mode 100644 index 000000000..300a6e641 --- /dev/null +++ b/test/test_wasm_imports.py @@ -0,0 +1,107 @@ +from pathlib import Path +import shutil +import subprocess + +import pytest + +from unit.applications.lang.wasm import ApplicationWasm +from unit.check.check_prerequisites import check_prerequisites +from unit.option import option + +prerequisites = {'modules': {'wasm': 'any'}, 'features': {'clang_wasm': True}} + +check_prerequisites(prerequisites) + +client = ApplicationWasm() + +# The request buffer starts at 1 MiB. The host asks for 32 MiB + 64 KiB +# from there, so the memory needs at least 33 MiB + 64 KiB. +INITIAL_MEMORY = 36 * 1024 * 1024 + + +def build_noreq_guest(call, initialize=False): + source = Path(option.test_dir) / 'wasm' / 'noreq' / 'noreq.c' + suffix = '-initialize' if initialize else '' + output = Path(option.temp_dir) / f'noreq-{call}{suffix}.wasm' + + defines = [f'-DCALL_{call.upper()}'] + + if initialize: + defines.append('-DCALL_IN_INITIALIZE') + + subprocess.check_output( + [ + shutil.which('clang'), + '--target=wasm32-unknown-unknown', + '-O2', + '-nostdlib', + *defines, + '-Wl,--no-entry,--stack-first,-z,stack-size=65536,' + f'--initial-memory={INITIAL_MEMORY}', + str(source), + '-o', + str(output), + ], + stderr=subprocess.STDOUT, + ) + + return output + + +def load_noreq(module, **handlers): + app = { + 'type': 'wasm', + 'processes': {'spare': 0}, + 'module': str(module), + 'request_handler': 'request_handler', + 'malloc_handler': 'malloc_handler', + 'free_handler': 'free_handler', + } + app.update(handlers) + + assert 'success' in client.conf( + { + 'listeners': {'*:8080': {'pass': 'applications/noreq'}}, + 'applications': {'noreq': app}, + } + ) + + +@pytest.mark.parametrize( + 'call', ['send_response', 'send_headers', 'response_end'] +) +def test_wasm_import_in_initialize(call, wait_for_record): + load_noreq(build_noreq_guest(call, initialize=True)) + + assert client.get()['status'] == 503 + + record = rf'nxt_wasm_{call}\(\) called outside a request' + assert wait_for_record(record) is not None + + +def test_wasm_import_in_malloc_handler(wait_for_record): + load_noreq( + build_noreq_guest('send_headers'), malloc_handler='malloc_import' + ) + + assert client.get()['status'] == 503 + + record = r'nxt_wasm_send_headers\(\) called outside a request' + assert wait_for_record(record) is not None + + +def test_wasm_import_in_module_init(wait_for_record): + load_noreq( + build_noreq_guest('response_end'), module_init_handler='init_import' + ) + + resp = client.get() + + assert resp['status'] == 200 + assert resp['body'] == 'ok\n' + + record = r'failed to call hook function \[init_import\]' + assert wait_for_record(record) is not None + + record = r'nxt_wasm_response_end\(\) called outside a request' + assert wait_for_record(record) is not None diff --git a/test/wasm/noreq/noreq.c b/test/wasm/noreq/noreq.c new file mode 100644 index 000000000..6ddd4452f --- /dev/null +++ b/test/wasm/noreq/noreq.c @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright (C) FreeUnit contributors. */ + +/* + * A guest that calls a host import before the host serves a request. It is + * built without libc or libunit-wasm. -DCALL_SEND_RESPONSE, + * -DCALL_SEND_HEADERS or -DCALL_RESPONSE_END selects the import. With + * -DCALL_IN_INITIALIZE, the guest exports "_initialize", and wasmtime calls + * it when it instantiates the module. "malloc_import" and "init_import" + * call the import from the malloc handler and from the module_init hook. + * The other handlers work, so a host that serves a request answers 200 with + * "ok". + */ + +typedef unsigned int u32; + +#define IMPORT(name) __attribute__((import_module("env"), import_name(name))) +#define EXPORT(name) __attribute__((export_name(name))) + +/* The request buffer starts above the data and the stack of the guest. */ +#define BUFFER_OFFSET (1024 * 1024) +#define RESPONSE_OFFSET 4096 + +IMPORT("nxt_wasm_send_response") void send_response(u32 offset); +IMPORT("nxt_wasm_send_headers") void send_headers(u32 offset); +IMPORT("nxt_wasm_response_end") void response_end(void); + +static void +call_import(void) +{ +#if defined(CALL_SEND_RESPONSE) + send_response(RESPONSE_OFFSET); +#elif defined(CALL_SEND_HEADERS) + send_headers(RESPONSE_OFFSET); +#elif defined(CALL_RESPONSE_END) + response_end(); +#else +#error "define CALL_SEND_RESPONSE, CALL_SEND_HEADERS or CALL_RESPONSE_END" +#endif +} + +#if defined(CALL_IN_INITIALIZE) +EXPORT("_initialize") void +initialize(void) +{ + call_import(); +} +#endif + +EXPORT("malloc_import") u32 +malloc_import(u32 size) +{ + (void) size; + + call_import(); + + return BUFFER_OFFSET; +} + +EXPORT("init_import") void +init_import(void) +{ + call_import(); +} + +EXPORT("malloc_handler") u32 +malloc_handler(u32 size) +{ + (void) size; + + return BUFFER_OFFSET; +} + +EXPORT("free_handler") void +free_handler(u32 addr) +{ + (void) addr; +} + +EXPORT("request_handler") int +request_handler(u32 addr) +{ + unsigned char *resp; + + resp = (unsigned char *) addr + RESPONSE_OFFSET; + + /* The layout of nxt_wasm_response_t: a u32 size, then the data. */ + *(u32 *) resp = 3; + resp[4] = 'o'; + resp[5] = 'k'; + resp[6] = '\n'; + + send_response(RESPONSE_OFFSET); + response_end(); + + return 0; +} From a033621e45edd2348004c0a81cd113d22e35189e Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 6 Oct 2026 17:05:18 +0200 Subject: [PATCH 383/391] controller: ignore a line end in the state version file The controller reads /version with nxt_int_parse(). That function returns -1 for any byte that is not a digit. On -1, nxt_controller_prefork() logs "failed to restore previous configuration: invalid version string" and drops conf.json, so unitd starts with no configuration. unitd writes the file with nxt_sprintf("%d") and no line end. A file written by hand usually has one: "echo 13700 > version" adds "\n", most editors add a final "\n", and a file from Windows has "\r\n". Each of these lost the whole stored configuration. The new nxt_controller_version_parse() skips trailing "\n", "\r", space and tab, and then calls nxt_int_parse(). Any other byte that is not a digit is still an error. An empty file, or a file with only white space, is still an error too. The file is not rewritten on load. The controller is the only reader of the version file. The behaviour dates from 1.26.0, which added the file. Found in the review of https://github.com/freeunitorg/freeunit/pull/586. test/test_state_store.py seeds a state directory and starts unitd with --restart. With "13700\n", "13700\r\n" and "13700 \t\n" the stored configuration now loads. With "13700x\n", "", "\n", " 13700" and "137\n00" unitd still logs the alert and restores nothing. Co-Authored-By: Claude Opus 5.5 --- CHANGES | 8 +++++ docs/changes.xml | 10 ++++++ src/nxt_controller.c | 32 +++++++++++++++++- test/test_state_store.py | 70 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 225c8c3a4..f7bdd05e8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ Changes with FreeUnit 1.37.1 04 Oct 2026 + *) Bugfix: unitd did not load the stored configuration when the + "version" file in the state directory ended with a line end, as + "echo 13700 > version" and most editors write it. It logged + "invalid version string" and started with no configuration. A line + end, a space or a tab after the number is now ignored. Any other + content is still an error. unitd itself writes the file with no + line end. The bug appeared in 1.26.0. + *) Change: "--otel" no longer needs OpenSSL. Before, configure stopped with "OpenTelemetry support requires OpenSSL." when the OpenSSL headers were missing, and it linked unitd with OpenSSL diff --git a/docs/changes.xml b/docs/changes.xml index 06ffa7937..eb061d4b6 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,16 @@ date="2026-10-04" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +unitd did not load the stored configuration when the "version" file in the +state directory ended with a line end, as "echo 13700 > version" and most +editors write it. It logged "invalid version string" and started with no +configuration. A line end, a space or a tab after the number is now ignored. +Any other content is still an error. The bug appeared in 1.26.0. + + + "--otel" no longer needs OpenSSL. Before, configure stopped with diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 4755a63f4..4de8e8418 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -65,6 +65,7 @@ static nxt_int_t nxt_controller_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp); static nxt_int_t nxt_controller_file_read(nxt_task_t *task, const char *name, nxt_str_t *str, nxt_mp_t *mp); +static nxt_int_t nxt_controller_version_parse(const nxt_str_t *ver); static nxt_int_t nxt_controller_start(nxt_task_t *task, nxt_process_data_t *data); static void nxt_controller_process_new_port_handler(nxt_task_t *task, @@ -236,7 +237,7 @@ nxt_controller_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) } if (ret == NXT_OK) { - num = nxt_int_parse(ver.start, ver.length); + num = nxt_controller_version_parse(&ver); if (nxt_slow_path(num < 0)) { nxt_alert(task, "failed to restore previous configuration: " @@ -320,6 +321,35 @@ nxt_controller_file_read(nxt_task_t *task, const char *name, nxt_str_t *str, } +/* + * unitd writes the version file with no line end. A file written by hand + * or by an editor often ends with "\n" or "\r\n". Skip trailing ASCII + * white space. Any other byte that is not a digit is still an error, and so + * is a file with no digits. + */ + +static nxt_int_t +nxt_controller_version_parse(const nxt_str_t *ver) +{ + u_char c; + size_t length; + + length = ver->length; + + while (length != 0) { + c = ver->start[length - 1]; + + if (c != '\n' && c != '\r' && c != ' ' && c != '\t') { + break; + } + + length--; + } + + return nxt_int_parse(ver->start, length); +} + + #if (NXT_TLS) static void diff --git a/test/test_state_store.py b/test/test_state_store.py index 9242b340d..4a36ba917 100644 --- a/test/test_state_store.py +++ b/test/test_state_store.py @@ -18,6 +18,9 @@ last configuration. They need --restart only. They catch a regression only when a store takes longer than the few milliseconds main needs to exit, so on a tmpfs they pass either way. + +The version tests seed a state directory by hand. unitd writes the version +file with no line end, but a file written by hand often has one. """ import json @@ -335,3 +338,70 @@ def test_state_store_serialised(requires_restart): finally: unit_stop() shutil.rmtree(statedir, ignore_errors=True) + + +VERSION_CONF = { + "listeners": {"*:8080": {"pass": "routes"}}, + "routes": [{"action": {"return": 204}}], +} + + +def run_with_version(version): + """Start unitd on a state directory with this version file.""" + + unit_stop() + + statedir = Path(tempfile.mkdtemp(prefix='unit-state-')) + + (statedir / 'conf.json').write_text(json.dumps(VERSION_CONF)) + (statedir / 'version').write_bytes(version) + + unit_run(state_dir=str(statedir)) + + return statedir + + +@pytest.mark.parametrize( + 'version', + [b'13700\n', b'13700\r\n', b'13700 \t\n'], + ids=['lf', 'crlf', 'ws'], +) +def test_state_store_version_line_end(requires_restart, version): + """A line end after the number does not drop the stored configuration.""" + + statedir = run_with_version(version) + + try: + assert ( + client.conf_get('listeners') == VERSION_CONF['listeners'] + ), 'stored configuration loaded' + assert client.get()['status'] == 204, 'stored configuration runs' + assert not Log.findall(r'invalid version string'), 'no alert' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) + + +@pytest.mark.parametrize( + 'version', + [b'13700x\n', b'', b'\n', b' 13700', b'137\n00'], + ids=['suffix', 'empty', 'blank', 'leading', 'inner'], +) +def test_state_store_version_invalid(requires_restart, skip_alert, version): + """Other content is still an error, and nothing is restored.""" + + skip_alert(r'failed to restore previous configuration') + + statedir = run_with_version(version) + + try: + assert Log.wait_for_record( + r'failed to restore previous configuration: ' + r'invalid version string' + ), 'alert' + assert client.conf_get('listeners') == {}, 'nothing restored' + + finally: + unit_stop() + shutil.rmtree(statedir, ignore_errors=True) From c8b33ba885884244bd39dba977d7c1db1d36b6ca Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 6 Oct 2026 17:00:22 +0200 Subject: [PATCH 384/391] http: consume chunk data that ends at the end of a read With "chunked_transform", a chunked request body is read into a temporary file. When a read ended right after the data of a chunk, nxt_http_chunk_buffer() returned NXT_HTTP_CHUNK_END_ON_BORDER and left in->mem.pos at the start of the data. The "between chunks" branch of nxt_h1p_conn_request_body_read() keeps the bytes from pos to free for the next read. So the data stayed in the buffer. The next read was parsed after it, the parser found a data byte where the CRLF must be, and the request got 400. Now the border case moves in->mem.pos to in->mem.free. The data is already linked into the output buffer, and the caller writes it to the file before it compacts the read buffer. https://github.com/freeunitorg/freeunit/issues/543 Co-Authored-By: Claude Opus 5.5 --- CHANGES | 6 ++++++ docs/changes.xml | 7 +++++++ src/nxt_http_chunk_parse.c | 7 +++++++ test/test_chunked.py | 41 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+) diff --git a/CHANGES b/CHANGES index f7bdd05e8..c63494232 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,12 @@ Changes with FreeUnit 1.37.1 04 Oct 2026 + *) Bugfix: with "chunked_transform", a valid chunked request body got + 400 when a read ended right after the data of a chunk, before the + CRLF after it. The data stayed in the read buffer, and the next read + was parsed after it. So the result depended on how the client's + writes reached the server. The bug had appeared in 1.35.5. + *) Bugfix: unitd did not load the stored configuration when the "version" file in the state directory ended with a line end, as "echo 13700 > version" and most editors write it. It logged diff --git a/docs/changes.xml b/docs/changes.xml index eb061d4b6..14c6270b4 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -10,6 +10,13 @@ date="2026-10-04" time="00:00:00 +0000" packager="FreeUnit Packaging <team@freeunit.org>"> + + +with "chunked_transform", a valid chunked request body got 400 when a read +ended right after the data of a chunk, before the CRLF after it. + + + unitd did not load the stored configuration when the "version" file in the diff --git a/src/nxt_http_chunk_parse.c b/src/nxt_http_chunk_parse.c index 559d704ae..a8c703c34 100644 --- a/src/nxt_http_chunk_parse.c +++ b/src/nxt_http_chunk_parse.c @@ -398,6 +398,13 @@ nxt_http_chunk_buffer(nxt_http_chunk_parse_t *hcp, nxt_buf_t ***tail, hcp->chunk_size -= size; if (hcp->chunk_size == 0) { + /* + * The whole chunk data is in the output buffer. Consume it, so + * that a caller that keeps the bytes after pos for the next read + * does not parse the data again as framing. + */ + in->mem.pos = in->mem.free; + return NXT_HTTP_CHUNK_END_ON_BORDER; } diff --git a/test/test_chunked.py b/test/test_chunked.py index eb9cb1e67..894a9808f 100644 --- a/test/test_chunked.py +++ b/test/test_chunked.py @@ -167,6 +167,47 @@ def test_chunked_malformed_later_read(): assert elapsed < 3, elapsed +def test_chunked_read_ends_after_chunk_data(): + # A read that ended right after the chunk data, before its CRLF, left + # that data in the read buffer. The next read was parsed after it, so + # the parser found "h" where the CRLF must be, and the request got 400. + # https://github.com/freeunitorg/freeunit/issues/543 + head = ( + b'POST / HTTP/1.1\r\nHost: localhost\r\n' + b'Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n' + ) + + def send(writes, expect_body): + sock = client.http(writes[0], raw=True, no_recv=True) + + # The pause puts each write in its own read. + for w in writes[1:]: + time.sleep(0.3) + sock.sendall(w) + + resp = client.recvall(sock, read_timeout=10) + sock.close() + + assert resp.startswith(b'HTTP/1.1 200 '), (writes, resp) + assert resp.endswith(b'\r\n\r\n' + expect_body), (writes, resp) + + # The read after the header read ends after the chunk data. + send([head, b'5\r\nhello', b'\r\n0\r\n\r\n'], b'hello') + + # The same, with a second chunk after the CRLF. + send( + [head, b'5\r\nhello', b'\r\n6\r\nworld!', b'\r\n0\r\n\r\n'], + b'helloworld!', + ) + + # The header read itself ends after the chunk data. + send([head + b'5\r\nhello', b'\r\n0\r\n\r\n'], b'hello') + + # Controls: the read ends inside the data, or after its CRLF. + send([head, b'5\r\nhel', b'lo\r\n0\r\n\r\n'], b'hello') + send([head, b'5\r\nhello\r\n', b'0\r\n\r\n'], b'hello') + + def test_chunked_after_last(): resp = client.get( headers={ From 7cfd11edcb3e27dda644ccf59182eb6bbe375234 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 6 Oct 2026 18:00:40 +0200 Subject: [PATCH 385/391] controller: parse the version file with nxt_str_strip() https://github.com/freeunitorg/freeunit/pull/587 added nxt_controller_version_parse(). It strips trailing CR, LF, SP and HTAB from the state version file, and then calls nxt_int_parse(). It has one caller. nxt_str_strip() in src/nxt_string.c already strips a line end, but it had no callers. It had two defects: - For an empty string it computed end - 1 before it compared the pointer with start. That pointer is before the buffer, which is undefined behavior. - It did not strip SP or HTAB. nxt_str_strip() now starts at end and stops at start, and it also strips SP and HTAB. end is now const. The controller calls nxt_int_parse() with the length from nxt_str_strip(), and nxt_controller_version_parse() is removed. The behavior of the controller does not change. The string test in src/test/nxt_string_test.c gets seven nxt_str_strip() cases. With the old body of nxt_str_strip() the test fails with 'nxt_str_strip("13700 \t\n") failed: got 7, expected 5'. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- src/nxt_controller.c | 35 ++++------------------------------- src/nxt_string.c | 12 +++++++----- src/nxt_string.h | 2 +- src/test/nxt_string_test.c | 27 +++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 4de8e8418..751570f0b 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -65,7 +65,6 @@ static nxt_int_t nxt_controller_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp); static nxt_int_t nxt_controller_file_read(nxt_task_t *task, const char *name, nxt_str_t *str, nxt_mp_t *mp); -static nxt_int_t nxt_controller_version_parse(const nxt_str_t *ver); static nxt_int_t nxt_controller_start(nxt_task_t *task, nxt_process_data_t *data); static void nxt_controller_process_new_port_handler(nxt_task_t *task, @@ -237,7 +236,10 @@ nxt_controller_prefork(nxt_task_t *task, nxt_process_t *process, nxt_mp_t *mp) } if (ret == NXT_OK) { - num = nxt_controller_version_parse(&ver); + /* unitd writes no line end, but an editor may add one. */ + num = nxt_int_parse(ver.start, + nxt_str_strip(ver.start, + ver.start + ver.length)); if (nxt_slow_path(num < 0)) { nxt_alert(task, "failed to restore previous configuration: " @@ -321,35 +323,6 @@ nxt_controller_file_read(nxt_task_t *task, const char *name, nxt_str_t *str, } -/* - * unitd writes the version file with no line end. A file written by hand - * or by an editor often ends with "\n" or "\r\n". Skip trailing ASCII - * white space. Any other byte that is not a digit is still an error, and so - * is a file with no digits. - */ - -static nxt_int_t -nxt_controller_version_parse(const nxt_str_t *ver) -{ - u_char c; - size_t length; - - length = ver->length; - - while (length != 0) { - c = ver->start[length - 1]; - - if (c != '\n' && c != '\r' && c != ' ' && c != '\t') { - break; - } - - length--; - } - - return nxt_int_parse(ver->start, length); -} - - #if (NXT_TLS) static void diff --git a/src/nxt_string.c b/src/nxt_string.c index 3f390d6ff..aa1c2a4fc 100644 --- a/src/nxt_string.c +++ b/src/nxt_string.c @@ -341,18 +341,20 @@ nxt_rmemstrn(const u_char *s, const u_char *end, const char *ss, size_t length) } +/* The length of the string without trailing CR, LF, SP and HTAB. */ + size_t -nxt_str_strip(const u_char *start, u_char *end) +nxt_str_strip(const u_char *start, const u_char *end) { - u_char *p; + const u_char *p; - for (p = end - 1; p >= start; p--) { - if (*p != '\r' && *p != '\n') { + for (p = end; p != start; p--) { + if (p[-1] != '\r' && p[-1] != '\n' && p[-1] != ' ' && p[-1] != '\t') { break; } } - return (p + 1) - start; + return p - start; } diff --git a/src/nxt_string.h b/src/nxt_string.h index 2b3741003..bfa667735 100644 --- a/src/nxt_string.h +++ b/src/nxt_string.h @@ -97,7 +97,7 @@ NXT_EXPORT u_char *nxt_memcasestrn(const u_char *s, const u_char *end, const char *ss, size_t length); NXT_EXPORT u_char *nxt_rmemstrn(const u_char *s, const u_char *end, const char *ss, size_t length); -NXT_EXPORT size_t nxt_str_strip(const u_char *start, u_char *end); +NXT_EXPORT size_t nxt_str_strip(const u_char *start, const u_char *end); typedef struct { diff --git a/src/test/nxt_string_test.c b/src/test/nxt_string_test.c index f82d19be4..51400858b 100644 --- a/src/test/nxt_string_test.c +++ b/src/test/nxt_string_test.c @@ -11,6 +11,7 @@ nxt_int_t nxt_string_test(nxt_thread_t *thr) { u_char *p; + size_t n; nxt_uint_t i; nxt_bool_t enc; @@ -51,6 +52,20 @@ nxt_string_test(nxt_thread_t *thr) { nxt_string(" "), 0 }, /* space must be escaped */ }; + /* nxt_str_strip(): the length without trailing CR, LF, SP and HTAB. */ + static const struct { + nxt_str_t str; + size_t length; + } strip[] = { + { nxt_string("13700"), 5 }, + { nxt_string("13700\r\n"), 5 }, + { nxt_string("13700 \t\n"), 5 }, + { nxt_string("1 2\n"), 3 }, /* inner space stays */ + { nxt_string(" 13700"), 6 }, /* leading space stays */ + { nxt_string(" \t\r\n"), 0 }, + { nxt_string(""), 0 }, + }; + nxt_thread_time_update(thr); for (i = 0; i < nxt_nitems(rmem); i++) { @@ -94,6 +109,18 @@ nxt_string_test(nxt_thread_t *thr) } } + for (i = 0; i < nxt_nitems(strip); i++) { + n = nxt_str_strip(strip[i].str.start, + strip[i].str.start + strip[i].str.length); + + if (n != strip[i].length) { + nxt_log_alert(thr->log, "nxt_str_strip(\"%V\") failed: " + "got %uz, expected %uz", &strip[i].str, n, + strip[i].length); + return NXT_ERROR; + } + } + nxt_log_error(NXT_LOG_NOTICE, thr->log, "nxt_string helpers test passed"); return NXT_OK; From 2e334334f6c52b6e0d7fce761079a0dba9076f68 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 6 Oct 2026 17:55:37 +0200 Subject: [PATCH 386/391] port: do not re-arm an active read event after each drain The port read handlers call nxt_fd_event_enable_read() when a read returns EAGAIN. On epoll, enable_read queues an EPOLL_CTL_MOD even when the read event is already active. So each drain of a port cost one epoll_ctl call. Under keep-alive load the router made about 4 of them per request. An active read event stays registered. With the default edge-triggered mode, the next datagram raises a new event. So the re-arm is needed only when the event is not active. Read leaves the active state in three places: nxt_port_read_close() makes it inactive, nxt_fd_event_block_read() on a buffer allocation failure makes it blocked, and the engine's level-triggered disable of a blocked read makes it inactive. All three still take the enable path. App processes use the libunit read loop, which does not call epoll_ctl, and are not changed. The ctrunc test now counts the enable_read calls of its stub engine, and the stub leaves the event active as a real engine does. Each of its three drains must make no call. The previous code makes one call per drain and fails the test. The router was traced with strace for 2000 keep-alive requests (oha -c 1, external test app), two runs per build: - epoll_ctl: 7998 and 7922 before, 2 and 2 after. - epoll_wait: 10049 and 10077 before, 6288 and 6294 after. Co-Authored-By: Claude Opus 5.5 Reviewed-by: DeepSeek V4.1 Flash --- src/nxt_port_socket.c | 17 +++++++++++++-- src/test/nxt_port_ctrunc_test.c | 37 ++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/nxt_port_socket.c b/src/nxt_port_socket.c index 542429310..537fc029d 100644 --- a/src/nxt_port_socket.c +++ b/src/nxt_port_socket.c @@ -1946,7 +1946,11 @@ nxt_port_read_handler(nxt_task_t *task, void *obj, void *data) if (n == NXT_AGAIN) { nxt_port_buf_free(port, b); - nxt_fd_event_enable_read(task->thread->engine, &port->socket); + /* See the comment in nxt_port_queue_read_handler(). */ + if (!nxt_fd_event_is_active(port->socket.read)) { + nxt_fd_event_enable_read(task->thread->engine, &port->socket); + } + return; } @@ -2248,7 +2252,16 @@ nxt_port_queue_read_handler(nxt_task_t *task, void *obj, void *data) if (n == NXT_AGAIN) { nxt_port_buf_free(port, b); - nxt_fd_event_enable_read(task->thread->engine, &port->socket); + /* + * An active read event stays registered, so EAGAIN needs no + * re-arm: the next datagram raises a new event. On epoll the + * enable is an EPOLL_CTL_MOD even when the event is active, and + * this point is reached once per drain of the port. Enable only + * an event that is not active. + */ + if (!nxt_fd_event_is_active(port->socket.read)) { + nxt_fd_event_enable_read(task->thread->engine, &port->socket); + } continue; } diff --git a/src/test/nxt_port_ctrunc_test.c b/src/test/nxt_port_ctrunc_test.c index 5b6fec913..16a032e87 100644 --- a/src/test/nxt_port_ctrunc_test.c +++ b/src/test/nxt_port_ctrunc_test.c @@ -587,6 +587,7 @@ nxt_port_ctrunc_test_marked(nxt_thread_t *thr, nxt_socket_t *pair) #if (NXT_HAVE_UCRED) static nxt_uint_t nxt_port_ctrunc_test_delivered; +static nxt_uint_t nxt_port_ctrunc_test_enables; static void @@ -605,7 +606,16 @@ static void nxt_port_ctrunc_test_enable_read_stub(nxt_event_engine_t *engine, nxt_fd_event_t *ev) { - /* The fixture engine polls nothing. */ + /* + * The fixture engine polls nothing. Count the calls, and leave the + * event active as a real engine does. A drain that ends with EAGAIN + * must not re-arm an active read event: on epoll that costs an + * epoll_ctl() call per drain. + */ + + nxt_port_ctrunc_test_enables++; + + ev->read = NXT_EVENT_ACTIVE; } @@ -755,6 +765,7 @@ nxt_port_ctrunc_test_queue(nxt_thread_t *thr) nxt_port_msg_t port_msg; nxt_port_queue_t *queue; nxt_event_engine_t *engine, *saved_engine; + nxt_uint_t enables; nxt_nncq_atomic_t nitems; task = thr->task; @@ -866,8 +877,16 @@ nxt_port_ctrunc_test_queue(nxt_thread_t *thr) nxt_port_ctrunc_test_delivered = 0; + enables = nxt_port_ctrunc_test_enables; + port->socket.read_handler(task, &port->socket, NULL); + if (nxt_port_ctrunc_test_enables != enables) { + nxt_log_alert(thr->log, "port ctrunc test: drain 1 re-armed " + "an active read event"); + goto done; + } + if (nxt_port_ctrunc_test_delivered != 0) { nxt_log_alert(thr->log, "port ctrunc test: a message with an " "unusable control block was delivered"); @@ -903,8 +922,16 @@ nxt_port_ctrunc_test_queue(nxt_thread_t *thr) port->socket.read_ready = 1; + enables = nxt_port_ctrunc_test_enables; + port->socket.read_handler(task, &port->socket, NULL); + if (nxt_port_ctrunc_test_enables != enables) { + nxt_log_alert(thr->log, "port ctrunc test: drain 2 re-armed " + "an active read event"); + goto done; + } + if (nxt_port_ctrunc_test_delivered != 1) { nxt_log_alert(thr->log, "port ctrunc test: the message after a " "refused one was not delivered"); @@ -951,8 +978,16 @@ nxt_port_ctrunc_test_queue(nxt_thread_t *thr) port->socket.read_ready = 1; + enables = nxt_port_ctrunc_test_enables; + port->socket.read_handler(task, &port->socket, NULL); + if (nxt_port_ctrunc_test_enables != enables) { + nxt_log_alert(thr->log, "port ctrunc test: drain 3 re-armed " + "an active read event"); + goto done; + } + if (nxt_port_ctrunc_test_delivered != 1) { nxt_log_alert(thr->log, "port ctrunc test: a message queued behind a " "refused one was stranded"); From 6b204e234ce4cde4c711432c519a05f04bded229 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Tue, 6 Oct 2026 19:40:49 +0200 Subject: [PATCH 387/391] tests: wait for the listener in the version file test test_state_store_version_line_end checked the loaded configuration with GET /config, then sent a request to the listener. GET /config can answer before the router has applied the stored configuration and bound the listener, so the request could fail with "Client can't connect to the server." The test now waits for port 8080 first. CHANGES and docs/changes.xml now say "The bug had appeared in 1.26.0.", as the other entries do. These are the review fixes for https://github.com/freeunitorg/freeunit/pull/587, which was merged before they were pushed. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- CHANGES | 2 +- docs/changes.xml | 2 +- test/test_state_store.py | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index c63494232..f577f48b4 100644 --- a/CHANGES +++ b/CHANGES @@ -13,7 +13,7 @@ Changes with FreeUnit 1.37.1 04 Oct 2026 "invalid version string" and started with no configuration. A line end, a space or a tab after the number is now ignored. Any other content is still an error. unitd itself writes the file with no - line end. The bug appeared in 1.26.0. + line end. The bug had appeared in 1.26.0. *) Change: "--otel" no longer needs OpenSSL. Before, configure stopped with "OpenTelemetry support requires OpenSSL." when the diff --git a/docs/changes.xml b/docs/changes.xml index 14c6270b4..dd3d8063d 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -23,7 +23,7 @@ unitd did not load the stored configuration when the "version" file in the state directory ended with a line end, as "echo 13700 > version" and most editors write it. It logged "invalid version string" and started with no configuration. A line end, a space or a tab after the number is now ignored. -Any other content is still an error. The bug appeared in 1.26.0. +Any other content is still an error. The bug had appeared in 1.26.0. diff --git a/test/test_state_store.py b/test/test_state_store.py index 4a36ba917..96b179e86 100644 --- a/test/test_state_store.py +++ b/test/test_state_store.py @@ -38,6 +38,7 @@ from conftest import unit_run, unit_stop from unit.applications.proto import ApplicationProto from unit.log import Log +from unit.utils import waitforsocket client = ApplicationProto() @@ -375,6 +376,10 @@ def test_state_store_version_line_end(requires_restart, version): assert ( client.conf_get('listeners') == VERSION_CONF['listeners'] ), 'stored configuration loaded' + + # GET /config can answer before the router has bound the listener. + waitforsocket(8080) + assert client.get()['status'] == 204, 'stored configuration runs' assert not Log.findall(r'invalid version string'), 'no alert' From 23074fa1cf8ad3fce03764efb8ef34bbe1e6356c Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 5 Oct 2026 03:02:03 +0200 Subject: [PATCH 388/391] otel: disable the OTLP exporter retries that opentelemetry-otlp 0.33 enabled opentelemetry-otlp 0.33 retries a failed export up to 3 times by default, with backoff from 100 ms to 1600 ms. Version 0.32 retried only with the experimental-http-retry and experimental-grpc-retry features, which FreeUnit does not enable. The update in https://github.com/freeunitorg/freeunit/pull/571 (ccf1f523) turned the retries on. The batch worker blocks on each export. While it retries, later batches wait in the queue. Set RetryPolicy::disabled() on the HTTP and gRPC builders, so a batch is sent once, as in 1.37.0. fake_otlp gets a --status option. It answers each export with that HTTP status, or with UNAVAILABLE for gRPC, and still counts and dumps the request. test_otel_export_not_retried uses it with 503 and checks one request per failed batch. On the previous code it fails with "4 export requests for 1 failed batches" for http and grpc. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- src/otel/src/lib.rs | 12 ++++++++++- test/fake_otlp/README.md | 2 ++ test/fake_otlp/src/grpc.rs | 13 +++++++++++- test/fake_otlp/src/main.rs | 31 +++++++++++++++++++++------- test/test_otel.py | 41 +++++++++++++++++++++++++++++++++++++- 5 files changed, 89 insertions(+), 10 deletions(-) diff --git a/src/otel/src/lib.rs b/src/otel/src/lib.rs index 5f2a54526..c4b6b26d0 100644 --- a/src/otel/src/lib.rs +++ b/src/otel/src/lib.rs @@ -20,7 +20,10 @@ use opentelemetry::trace::{ TraceId, TraceState, Tracer, TracerProvider, }; use opentelemetry::{Context, Key, KeyValue, Value}; -use opentelemetry_otlp::{Protocol, SpanExporter, WithExportConfig}; +use opentelemetry_otlp::{ + Protocol, RetryPolicy, SpanExporter, WithExportConfig, WithHttpConfig, + WithTonicConfig, +}; use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::trace::{ BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider, SpanData, @@ -213,12 +216,18 @@ impl SdkSpanExporter for FailureTrackingExporter { } /// Build the OTLP/HTTP exporter: the blocking reqwest client, no async runtime. +/// +/// opentelemetry-otlp 0.33 retries a failed export up to 3 times by default. +/// Both builders disable this. The batch worker blocks on each export, so the +/// retries would hold the queue while the collector fails. A failed batch is +/// counted as failed in /status and dropped. fn build_http_exporter(endpoint: String) -> Result { SpanExporter::builder() .with_http() .with_endpoint(endpoint) .with_protocol(Protocol::HttpBinary) .with_timeout(EXPORT_TIMEOUT) + .with_retry_policy(RetryPolicy::disabled()) .build() .map_err(|e| format!("couldn't build otel http exporter: {e}")) } @@ -240,6 +249,7 @@ fn build_grpc_exporter(endpoint: String) -> Result { .with_tonic() .with_endpoint(endpoint) .with_timeout(EXPORT_TIMEOUT) + .with_retry_policy(RetryPolicy::disabled()) .build() .map_err(|e| format!("couldn't build otel grpc exporter: {e}"))? }; diff --git a/test/fake_otlp/README.md b/test/fake_otlp/README.md index f8c0bfdcc..030ec47e3 100644 --- a/test/fake_otlp/README.md +++ b/test/fake_otlp/README.md @@ -27,6 +27,7 @@ built — mirroring FreeUnit's single `--otel` build. See PLAN.md. ``` fake_otlp --port [--protocol http|grpc] [--requests ] [--dump ] + [--status ] ``` | Flag | Meaning | @@ -35,6 +36,7 @@ fake_otlp --port [--protocol http|grpc] [--requests ] [--dump ] | `--protocol P` | transport (table below); default `http` | | `--requests N` | exit after N export requests (default: run forever) | | `--dump FILE` | append each received request (raw bytes) to FILE | +| `--status CODE` | answer each valid export with this HTTP status (default 200). For `grpc`, any other code answers `UNAVAILABLE`. The export is still counted and dumped. | ### Protocols diff --git a/test/fake_otlp/src/grpc.rs b/test/fake_otlp/src/grpc.rs index 6566ff25e..08c7a4acb 100644 --- a/test/fake_otlp/src/grpc.rs +++ b/test/fake_otlp/src/grpc.rs @@ -27,6 +27,7 @@ struct Collector { dump: Option, count: Arc, max_requests: Option, + status: u16, shutdown: Arc, } @@ -59,13 +60,22 @@ impl TraceService for Collector { self.shutdown.notify_one(); } + if self.status != 200 { + return Err(Status::unavailable("fake_otlp: --status failure")); + } + Ok(Response::new(ExportTraceServiceResponse::default())) } } /// Serve OTLP/gRPC until `max_requests` exports have arrived (or forever when /// `None`). Runs on a multi-thread tokio runtime owned by this thread. -pub fn serve_grpc(port: u16, max_requests: Option, dump: Option) { +pub fn serve_grpc( + port: u16, + max_requests: Option, + dump: Option, + status: u16, +) { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() @@ -77,6 +87,7 @@ pub fn serve_grpc(port: u16, max_requests: Option, dump: Option) dump, count: Arc::new(AtomicUsize::new(0)), max_requests, + status, shutdown: shutdown.clone(), }; diff --git a/test/fake_otlp/src/main.rs b/test/fake_otlp/src/main.rs index bf3ef18ad..3d73651f0 100644 --- a/test/fake_otlp/src/main.rs +++ b/test/fake_otlp/src/main.rs @@ -5,12 +5,16 @@ /// /// Usage: /// fake_otlp --port [--protocol http|grpc] [--requests ] [--dump ] +/// [--status ] /// /// --port N TCP port to listen on (127.0.0.1) /// --protocol P transport: "http" (default) or "grpc" /// (HTTP/2 unary TraceService/Export) /// --requests N exit after receiving N export requests (default: forever) /// --dump FILE append each received request to FILE +/// --status CODE answer each valid export with this HTTP status (default +/// 200); for grpc, any code other than 200 answers UNAVAILABLE. +/// The export is still counted and dumped. /// /// HTTP behaviour: /// * Accepts POST /v1/traces with Content-Type: application/x-protobuf and a @@ -150,7 +154,7 @@ pub fn dump_request(dump: Option<&str>, bytes: &[u8]) { /// received, so a bare TCP probe (the test harness's readiness check, which /// connects and immediately closes) and malformed requests are not counted /// against --requests and are not written to the dump. -fn handle(mut stream: TcpStream, dump: Option<&str>) -> bool { +fn handle(mut stream: TcpStream, dump: Option<&str>, status: u16) -> bool { let req = match read_request(&mut stream) { Ok(r) => r, Err(_) => return false, @@ -185,13 +189,17 @@ fn handle(mut stream: TcpStream, dump: Option<&str>) -> bool { ); let _ = std::io::stdout().flush(); - respond(&mut stream, "200 OK"); + if status == 200 { + respond(&mut stream, "200 OK"); + } else { + respond(&mut stream, &format!("{status} Fake Failure")); + } true } /// Serve OTLP/HTTP until `max_requests` valid exports have been received (or /// forever when `None`). -fn serve_http(port: u16, max_requests: Option, dump: Option<&str>) { +fn serve_http(port: u16, max_requests: Option, dump: Option<&str>, status: u16) { let listener = TcpListener::bind((HOST, port)).unwrap_or_else(|e| { eprintln!("bind {HOST}:{port} — {e}"); process::exit(1); @@ -201,7 +209,7 @@ fn serve_http(port: u16, max_requests: Option, dump: Option<&str>) { for stream in listener.incoming() { match stream { Ok(s) => { - if handle(s, dump) { + if handle(s, dump, status) { count += 1; if max_requests.map_or(false, |n| count >= n) { break; @@ -221,7 +229,7 @@ fn serve_http(port: u16, max_requests: Option, dump: Option<&str>) { fn usage() -> ! { eprintln!( "Usage: fake_otlp --port [--protocol http|grpc] \ - [--requests ] [--dump ]" + [--requests ] [--dump ] [--status ]" ); process::exit(1); } @@ -233,6 +241,7 @@ fn main() { let mut protocol = String::from("http"); let mut max_requests: Option = None; let mut dump: Option = None; + let mut status: u16 = 200; let mut i = 1; while i < args.len() { @@ -255,6 +264,14 @@ fn main() { i += 1; dump = args.get(i).cloned(); } + "--status" => { + i += 1; + status = args + .get(i) + .and_then(|v| v.parse().ok()) + .filter(|v| (200..=599).contains(v)) + .unwrap_or_else(|| usage()); + } _ => {} } i += 1; @@ -263,8 +280,8 @@ fn main() { let port = port.unwrap_or_else(|| usage()); match protocol.as_str() { - "http" => serve_http(port, max_requests, dump.as_deref()), - "grpc" => grpc::serve_grpc(port, max_requests, dump), + "http" => serve_http(port, max_requests, dump.as_deref(), status), + "grpc" => grpc::serve_grpc(port, max_requests, dump, status), other => { eprintln!("fake_otlp: unknown --protocol {other:?} (use http or grpc)"); process::exit(1); diff --git a/test/test_otel.py b/test/test_otel.py index 998fa1080..4e4eb3730 100644 --- a/test/test_otel.py +++ b/test/test_otel.py @@ -45,8 +45,12 @@ def _get_free_port(): return s.getsockname()[1] -def _run_fake_otlp(port, requests=None, dump=None, protocol='http'): +def _run_fake_otlp( + port, requests=None, dump=None, protocol='http', status=None +): cmd = [FAKE_OTLP_BIN, '--port', str(port), '--protocol', protocol] + if status is not None: + cmd += ['--status', str(status)] if requests is not None: cmd += ['--requests', str(requests)] if dump is not None: @@ -928,3 +932,38 @@ def test_otel_status_counts_failed_spans(): assert telemetry is not None, '/status must report telemetry when configured' assert telemetry['spans']['failed'] > 0, 'a refused export must be counted' assert telemetry['spans']['exported'] == 0, 'nothing can have been exported' + + +@_skipif_no_fake_otlp +@pytest.mark.parametrize('protocol', ['http', 'grpc']) +def test_otel_export_not_retried(tmp_path, protocol): + """A batch the collector refuses with a retryable error is sent once. + + opentelemetry-otlp 0.33 retries such an export up to 3 times by default. + The batch worker blocks during the retries, so FreeUnit turns them off. + With batch_size 1 each failed span is one batch, so the collector must + see exactly one request per failed span. + """ + port = _get_free_port() + dump = tmp_path / 'otlp_dump.bin' + proc = _run_fake_otlp(port, dump=str(dump), protocol=protocol, status=503) + try: + _configure_or_skip(port, protocol=protocol) + + assert _get_until_header('traceparent')['status'] == 200 + + telemetry = _wait_for_spans('failed') + assert telemetry is not None, '/status must report telemetry' + assert telemetry['spans']['failed'] > 0, 'a 503 export must fail' + + # With retries on, the last attempt of a batch comes about 1 s after + # the first. Wait longer than that, then compare the counts. + time.sleep(4) + + failed = _status_telemetry()['spans']['failed'] + requests = dump.read_bytes().count(b'--fake_otlp-request-boundary--') + assert requests == failed, ( + f'{requests} export requests for {failed} failed batches' + ) + finally: + _kill(proc) From 7d729582908b487c275b7a776dc23fddff40ec00 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Mon, 5 Oct 2026 05:53:59 +0200 Subject: [PATCH 389/391] http: count large header buffers in a size_t The h1proto field nbuffers was a uint8_t. It counts the large buffers of a request header and is compared with the size_t setting "large_header_buffers". After 255 buffers the count went back to 0. With "large_header_buffers" set to 256 or more, the limit never fired. The parser rewinds only to the start of an incomplete field. So a header made of many short fields takes a new large buffer every few fields, and each buffer stays allocated until the request ends. With the limit off, the memory of such a header was bounded only by "header_read_timeout". Only an administrator can set the value. The counter is now a size_t, the type of the setting. It stops at the limit, so it cannot wrap first. nxt_h1proto_t grows from 704 to 712 bytes on x86_64 because of padding after the field. The 8 bytes per connection do not matter. A uint32_t would avoid the padding, but a limit above 2^32 - 1 would then depend on memory running out first. The setting is not capped at 255. The setting became valid in the configuration in 1.29.0. Before that, the validator rejected it, and the router used its default of 4. Found while reviewing https://github.com/freeunitorg/freeunit/pull/583. Co-Authored-By: Claude Opus 5.5 Reviewed-by: Claude Fable 5.1 --- CHANGES | 6 ++++++ docs/changes.xml | 9 +++++++++ src/nxt_h1proto.h | 2 +- test/test_http_header.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index f577f48b4..db2d177f7 100644 --- a/CHANGES +++ b/CHANGES @@ -34,6 +34,12 @@ Changes with FreeUnit 1.37.1 04 Oct 2026 a memory, the worker crashed on the first call of the handler. The module now refuses such a guest when the worker starts. + *) Bugfix: with "large_header_buffers" set to 256 or more, the limit + did not work. The router counted the large buffers of a request + header in 8 bits, so the count went back to 0 after 255. A request + header with many fields could then take memory until + "header_read_timeout" ended it. The bug had appeared in 1.29.0. + Changes with FreeUnit 1.37.0 04 Oct 2026 diff --git a/docs/changes.xml b/docs/changes.xml index dd3d8063d..043d4cce5 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -57,6 +57,15 @@ worker starts. + + +with "large_header_buffers" set to 256 or more, the limit did not work. The +router counted the large buffers of a request header in 8 bits, so the count +went back to 0 after 255. A request header with many fields could then take +memory until "header_read_timeout" ended it. The bug had appeared in 1.29.0. + + + diff --git a/src/nxt_h1proto.h b/src/nxt_h1proto.h index c4f3819e4..6a45f9f54 100644 --- a/src/nxt_h1proto.h +++ b/src/nxt_h1proto.h @@ -32,7 +32,7 @@ struct nxt_h1proto_s { nxt_http_chunk_parse_t chunked_parse; nxt_off_t remainder; - uint8_t nbuffers; + size_t nbuffers; uint8_t header_buffer_slot; uint8_t large_buffer_slot; uint8_t keepalive; /* 1 bit */ diff --git a/test/test_http_header.py b/test/test_http_header.py index fce5d6f60..286e492b5 100644 --- a/test/test_http_header.py +++ b/test/test_http_header.py @@ -549,3 +549,34 @@ def test_http_header_fields_keepalive_spill(): assert f'HTTP_X_TEST_{i}' in got, f'keep-alive field {i} of {count}' sock.close() + + +def test_http_header_large_buffers_above_255(): + # The parser rewinds only to the start of an incomplete field, so a + # header of many short fields takes one large buffer for every few + # fields. The first buffer has the default size of 2048 bytes and holds + # about 68 fields. With 128-byte large buffers and 30-byte fields, 4 + # fields fit in one buffer, so 1200 fields take about 283 large buffers + # and 1350 fields take about 320. The count of + # buffers was 8 bits wide, so a limit of 256 or more never fired. + client.load('empty') + + assert 'success' in client.conf( + { + 'http': { + 'large_header_buffer_size': 128, + 'large_header_buffers': 300, + } + }, + 'settings', + ) + + def fields(count, expect): + headers = {'Host': 'localhost', 'Connection': 'close'} + for i in range(count): + headers[f'X-{i:04}'] = 'a' * 20 + + assert client.get(headers=headers)['status'] == expect, count + + fields(1200, 200) + fields(1350, 431) From f09f5c4bd59e99106225eeabb204a030d7eb95e8 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Wed, 7 Oct 2026 07:28:49 +0200 Subject: [PATCH 390/391] test: name the descriptor kinds a leak assert found The descriptor check compared only counts, so a failure said "descriptors leak router" and nothing about what leaked. The Alpine aarch64 build runs with --fds-threshold=160 and #1129 is still open because nobody could tell which descriptors they were. Each baseline now also records the kinds behind the count, read from /proc//fd: pipe, memfd, the anon_inode name (eventfd, signalfd, eventpoll, io_uring), the file path, and for sockets the protocol found by inode in /proc//net: "tcp listen", "tcp close-wait", "udp6", "netlink", "unix" or "unix:". Kernel autobind names are folded into "unix:@autobind" so that equal sockets add up. A failing assert lists what changed: descriptors leak router: eventfd +1, unix +1 The threshold and the counting are unchanged. Without /proc the message says so. Co-Authored-By: exo-free --- test/conftest.py | 151 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 139 insertions(+), 12 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index b2ca6bee5..702808967 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,6 +10,7 @@ import sys import tempfile import time +from collections import Counter from multiprocessing import Process from pathlib import Path @@ -82,12 +83,19 @@ def pytest_addoption(parser): # unitd tree. _pgids = {} _fds_info = { - 'main': {'fds': 0, 'skip': False}, - 'router': {'name': 'unit: router', 'pid': -1, 'fds': 0, 'skip': False}, + 'main': {'fds': 0, 'kinds': {}, 'skip': False}, + 'router': { + 'name': 'unit: router', + 'pid': -1, + 'fds': 0, + 'kinds': {}, + 'skip': False, + }, 'controller': { 'name': 'unit: controller', 'pid': -1, 'fds': 0, + 'kinds': {}, 'skip': False, }, } @@ -224,11 +232,11 @@ def run(request): # lazy initialization (e.g. OTel tokio runtime) that occurred after # unit_run() captured the initial baseline is not counted as a leak. if not option.restart: - _fds_info['main']['fds'] = _count_fds(unit_instance['pid']) + _fd_baseline(_fds_info['main'], unit_instance['pid']) router = _fds_info['router'] - router['fds'] = _count_fds(router['pid']) + _fd_baseline(router, router['pid']) controller = _fds_info['controller'] - controller['fds'] = _count_fds(controller['pid']) + _fd_baseline(controller, controller['pid']) yield @@ -581,15 +589,15 @@ def unit_run(state_dir=None): if state_dir is None: _clear_conf() - _fds_info['main']['fds'] = _count_fds(unit_instance['pid']) + _fd_baseline(_fds_info['main'], unit_instance['pid']) router = _fds_info['router'] router['pid'] = pid_by_name(router['name']) - router['fds'] = _count_fds(router['pid']) + _fd_baseline(router, router['pid']) controller = _fds_info['controller'] controller['pid'] = pid_by_name(controller['name']) - controller['fds'] = _count_fds(controller['pid']) + _fd_baseline(controller, controller['pid']) Status._check_zeros() @@ -843,11 +851,14 @@ def waitforfds(diff): lambda: _count_fds(unit_instance['pid']) - ps['fds'] ) ps['fds'] += fds_diff + before, ps['kinds'] = ps['kinds'], _fd_kinds(unit_instance['pid']) - assert fds_diff <= option.fds_threshold, 'descriptors leak main process' + assert ( + fds_diff <= option.fds_threshold + ), f'descriptors leak main process: {_fd_kinds_diff(before, ps["kinds"])}' else: - ps['fds'] = _count_fds(unit_instance['pid']) + _fd_baseline(ps, unit_instance['pid']) for name in ['controller', 'router']: ps = _fds_info[name] @@ -857,14 +868,130 @@ def waitforfds(diff): if not ps['skip']: fds_diff = waitforfds(lambda: _count_fds(ps['pid']) - ps['fds']) ps['fds'] += fds_diff + before, ps['kinds'] = ps['kinds'], _fd_kinds(ps['pid']) if not option.restart: assert ps['pid'] == ps_pid, f'same pid {name}' - assert fds_diff <= option.fds_threshold, f'descriptors leak {name}' + assert ( + fds_diff <= option.fds_threshold + ), f'descriptors leak {name}: {_fd_kinds_diff(before, ps["kinds"])}' else: - ps['fds'] = _count_fds(ps['pid']) + _fd_baseline(ps, ps['pid']) + + +def _fd_baseline(ps, pid): + ps['fds'] = _count_fds(pid) + ps['kinds'] = _fd_kinds(pid) + + +_TCP_STATES = { + '01': 'established', + '02': 'syn-sent', + '03': 'syn-recv', + '04': 'fin-wait1', + '05': 'fin-wait2', + '07': 'close', + '08': 'close-wait', + '09': 'last-ack', + '0A': 'listen', + '0B': 'closing', +} + + +def _socket_names(pid): + names = {} + + for proto in ('tcp', 'tcp6', 'udp', 'udp6', 'netlink', 'unix'): + try: + lines = ( + Path(f'/proc/{pid}/net/{proto}') + .read_text(encoding='utf-8') + .splitlines()[1:] + ) + except OSError: + continue + + for line in lines: + f = line.split() + + if proto == 'unix': + if len(f) < 7: + continue + path = f[7] if len(f) > 7 else '' + if re.fullmatch(r'@[0-9a-f]{5}', path): + path = '@autobind' + names[f[6]] = f'unix:{path}' if path else 'unix' + + elif len(f) > 9: + if proto.startswith('tcp'): + name = f'{proto} {_TCP_STATES.get(f[3], f[3])}' + else: + name = proto + names[f[9]] = name + + return names + + +def _fd_kind(target, sockets): + if target.startswith('socket:'): + return sockets.get(target[len('socket:') :].strip('[]'), 'socket') + + for prefix, kind in ( + ('pipe:', 'pipe'), + ('/memfd:', 'memfd'), + ): + if target.startswith(prefix): + return kind + + if target.startswith('anon_inode:'): + return target[len('anon_inode:') :].strip('[]') + + return target + + +def _fd_kinds(pid): + kinds = Counter() + procfile = Path(f'/proc/{pid}/fd') + + try: + fds = list(procfile.iterdir()) + except OSError: + return kinds + + targets = [] + + for fd in fds: + try: + targets.append(os.readlink(fd)) + except OSError: + pass + + sockets = ( + _socket_names(pid) + if any(t.startswith('socket:') for t in targets) + else {} + ) + + for target in targets: + kinds[_fd_kind(target, sockets)] += 1 + + return kinds + + +def _fd_kinds_diff(before, after): + if not before and not after: + return 'no /proc fd detail' + + delta = Counter(after) + delta.subtract(before) + changed = sorted((k, n) for k, n in delta.items() if n) + + if not changed: + return 'same descriptor kinds as the baseline' + + return ', '.join(f'{k} {n:+d}' for k, n in changed) def _count_fds(pid): From cee5ceb0ea18bddc73b8625364b00c65e9d72d35 Mon Sep 17 00:00:00 2001 From: Andy Postnikov Date: Thu, 8 Oct 2026 02:12:15 +0200 Subject: [PATCH 391/391] test: fix two fd-kind parsing bugs found by review UnicodeDecodeError from read_text on /proc/net/unix crashed teardown when a socket path had non-UTF-8 bytes; unix paths with spaces were truncated by line.split(). Read bytes and decode with backslashreplace, and split unix lines with maxsplit=7 so the path survives spaces. Co-Authored-By: exo-free --- test/conftest.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 702808967..e8f9e13f2 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -853,9 +853,9 @@ def waitforfds(diff): ps['fds'] += fds_diff before, ps['kinds'] = ps['kinds'], _fd_kinds(unit_instance['pid']) - assert ( - fds_diff <= option.fds_threshold - ), f'descriptors leak main process: {_fd_kinds_diff(before, ps["kinds"])}' + assert fds_diff <= option.fds_threshold, _fd_leak( + 'main process', fds_diff, before, ps['kinds'] + ) else: _fd_baseline(ps, unit_instance['pid']) @@ -873,9 +873,9 @@ def waitforfds(diff): if not option.restart: assert ps['pid'] == ps_pid, f'same pid {name}' - assert ( - fds_diff <= option.fds_threshold - ), f'descriptors leak {name}: {_fd_kinds_diff(before, ps["kinds"])}' + assert fds_diff <= option.fds_threshold, _fd_leak( + name, fds_diff, before, ps['kinds'] + ) else: _fd_baseline(ps, ps['pid']) @@ -907,14 +907,15 @@ def _socket_names(pid): try: lines = ( Path(f'/proc/{pid}/net/{proto}') - .read_text(encoding='utf-8') + .read_bytes() + .decode('utf-8', 'backslashreplace') .splitlines()[1:] ) except OSError: continue for line in lines: - f = line.split() + f = line.split(maxsplit=7 if proto == 'unix' else -1) if proto == 'unix': if len(f) < 7: @@ -980,6 +981,11 @@ def _fd_kinds(pid): return kinds +def _fd_leak(name, fds_diff, before, after): + kinds = _fd_kinds_diff(before, after) + return f'descriptors leak {name}: {fds_diff:+d} fds, {kinds}' + + def _fd_kinds_diff(before, after): if not before and not after: return 'no /proc fd detail'